diff options
| author | John Klug <john.klug@multitech.com> | 2022-05-25 17:12:18 -0500 |
|---|---|---|
| committer | John Klug <john.klug@multitech.com> | 2022-05-25 17:12:18 -0500 |
| commit | 7971cb0aa3e517a53f0ce6d3ee9bc3179041ccb8 (patch) | |
| tree | 56e91417e6f937b3956cadcf4973756cebb4a8b0 | |
| download | mlinux-7971cb0aa3e517a53f0ce6d3ee9bc3179041ccb8.tar.gz mlinux-7971cb0aa3e517a53f0ce6d3ee9bc3179041ccb8.tar.bz2 mlinux-7971cb0aa3e517a53f0ce6d3ee9bc3179041ccb8.zip | |
mLinux 6
378 files changed, 39692 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9c02cf7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +downloads +build +buildhistory +env-oe.sh +layers/user-layer +bitbake.lock +password.txt diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..734426c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,57 @@ +[submodule "bitbake"] + path = bitbake + url = git://git.openembedded.org/bitbake + branch = 1.46 +[submodule "layers/openembedded-core"] + path = layers/openembedded-core + url = git://git.openembedded.org/openembedded-core + branch = dunfell +[submodule "layers/meta-openembedded"] + path = layers/meta-openembedded + url = git://git.openembedded.org/meta-openembedded + branch = dunfell +[submodule "layers/meta-multitech"] + path = layers/meta-multitech + url = git@gitlab.multitech.net:mirrors/meta-multitech.git + branch = . +[submodule "layers/meta-multitech-atmel"] + path = layers/meta-multitech-atmel + url = git@gitlab.multitech.net:mirrors/meta-multitech-atmel.git + branch = . +[submodule "layers/meta-java"] + path = layers/meta-java + branch = dunfell + url = git://git.yoctoproject.org/meta-java +[submodule "layers/meta-nodejs"] + path = layers/meta-nodejs + url = https://github.com/imyller/meta-nodejs.git + branch = master +[submodule "layers/meta-python2"] + path = layers/meta-python2 + url = git://git.openembedded.org/meta-python2 + branch = dunfell +[submodule "layers/meta-mono"] + path = layers/meta-mono + url = git://git.yoctoproject.org/meta-mono + branch = dunfell +[submodule "layers/meta-mlinux"] + path = layers/meta-mlinux + url = git@gitlab.multitech.net:mirrors/meta-mlinux.git + branch = . +[submodule "layers/meta-mlinux-atmel"] + path = layers/meta-mlinux-atmel + url = git@gitlab.multitech.net:mirrors/meta-mlinux-atmel.git + branch = . +[submodule "layers/meta-virtualization"] + path = layers/meta-virtualization + url = git://git.yoctoproject.org/meta-virtualization + branch = dunfell +[submodule "layers/meta-swupdate"] + path = layers/meta-swupdate + url = https://github.com/sbabic/meta-swupdate + branch = dunfell +[submodule "layers/meta-security"] + path = layers/meta-security + url = https://git.yoctoproject.org/git/meta-security + branch = dunfell + diff --git a/.templateconf b/.templateconf new file mode 100644 index 0000000..2d51eee --- /dev/null +++ b/.templateconf @@ -0,0 +1,2 @@ +# Template settings +TEMPLATECONF=${TEMPLATECONF:-layers/meta-mlinux/contrib/} @@ -0,0 +1,27 @@ +MultiTech mLinux is a software build system and custom Linux distribution. +It is your responsibility to review each license of the contained products +to understand what is, and what is not, allowed by each such license. + +The OpenEmbedded-Core common-licenses directory 'layers/openembedded-core/meta/files/common-licenses/' +contains licenses represented within this distribution. To the extent a license +is not contained in the 'common-licenses' directory, it is referenced in the +software and remains your responsibility to review. + +Different components of OpenEmbedded-Core are under different licenses (a mix +of MIT and GPLv2). Please see: + +layers/openembedded-core/meta/COPYING.GPLv2 (GPLv2) +layers/openembedded-core/meta/COPYING.MIT (MIT) + +For licensing information for other layers, see the files included with +that particular layer in the layers subdirectory. + +All MultiTech-copyrighted software contained in mLinux is covered by +GPL (GNU General Public License) version 2, unless provided under a different license. + +All bitbake metadata is MIT licensed unless otherwise stated. Source code +included in tree for individual recipes is under the LICENSE stated in +the associated recipe (.bb file) unless otherwise stated. + +License information for any other files is either explicitly stated +or defaults to GPL version 2. @@ -0,0 +1,121 @@ +This is the branch for mLinux >= 6.0.0
+
+mLinux - MultiTech Linux for MultiConnect Conduit gateways
+============================================================
+
+mLinux is a distribution provided by MultiTech tailored for the
+MultiConnect Conduit products (MTCDT models).
+
+mLinux utilizes OpenEmbedded-Core, Bitbake, which are part of the Yocto Project.
+
+For more information about mLinux, see http://www.multitech.net
+
+For more information about the Yocto Project, see http://www.yoctoproject.org
+For more information about OpenEmbedded, see http://www.openembedded.org
+
+This branch is based on Yocto Thud. Atmel devices use kernel Linux-AT91 4.9.
+
+It is no longer safe to assume MTCDT for the MACHINE type. All builds
+now require the external variable MACHINE to be set.
+
+Building MTCDTIP2 requires:
+
+MACHINE=mtcap
+
+
+-------------------------
+ BUILDING
+-------------------------
+
+All version between 5.1 and 5.3.0a inclusive have
+cellular issues with modems becoming disabled
+due to COPS=2 set more or less permanently
+on rare occassions.
+
+Version 5.3.0a through 5.3.0c have issues with git
+trying to access unreachable repositories. The
+recommended version after 5.0.x is 5.3.0d.
+
+Note that mLinux should *not* be built as root and you will need full write
+permissions to the build directory structure. It is recommended to build
+the software inside your $HOME dir as a normal user.
+
+1. Build from tarball
+
+ tar xzf mLinux-3.0.0.tar.gz
+ cd mlinux-3.0.0
+ # install needed dependencies, see scripts in install-deps
+ # Also switch to the build directory.
+ # example: sudo ./install-deps/install-debian-ubuntu-deps.sh
+ source oe-init-build-env
+
+ # set your default machine type in conf/local.conf
+ # MACHINE="mtcdt"
+
+ # build an image
+ bitbake mlinux-base-image
+
+2. Build from git repository
+
+ # clone repo to a dir name of your choice
+ git clone git://git.multitech.net/mlinux.git mlinux-3.0.0
+ cd mlinux-3.0.0
+
+ # checkout desired branch or tag
+ git checkout 3.0.0
+
+ # install needed dependencies, see scripts in install-deps
+ # example: sudo ./install-deps/install-debian-ubuntu-deps.sh
+
+ # initialize git submodules and setup dir structure
+ ./setup.sh
+ # setup.sh generates a random mtadm password, and places the
+ # password in build/conf/local.conf and password.txt
+ #
+ #
+ # The specified password is ignored for commissioning images.
+ #
+ # To specify a password use an environmental variable,
+ # MTADM_PASSWORD
+ MTADM_PASSWORD="3g_t1zX0" ./setup.sh
+ #
+ # To change the password, remove MTADM_PASSWORD and
+ # MTADM_PASSWORD_HASH from conf/local.conf and re-run:
+ MTADM_PASSWORD="Y5bG3m_2" ./setup.sh
+
+ # setup environment and switch to build directory
+ source oe-init-build-env
+
+ # set your default machine type in conf/local.conf
+ # MACHINE="mtcdt"
+
+ # build an image (current working directory is
+ # build)
+ bitbake mlinux-base-image
+
+-------------------------
+ IMAGES
+-------------------------
+
+mlinux-base-image: Small image with essential packages -- good starting point for a custom image
+
+mlinux-commissioning-image: Large image loaded on units by default -- contains full language support (perl, python)
+mlinux-mtcap-commissioning-image: MTCAP image
+
+-------------------------
+ PRE-BUILT IMAGES
+-------------------------
+
+http://multitech.net/mlinux/images
+
+-------------------------
+ MORE INFO
+-------------------------
+
+See the MultiTech Developer website (http://www.multitech.net) for full
+installation and build instructions. In particular, see the "Introduction" and
+"Getting Started" pages under the mLinux section.
+
+You can also join the available developer forums to post queries and receive
+updates from MultiTech developers.
+
diff --git a/bitbake b/bitbake new file mode 160000 +Subproject 0e0af15b84e07e6763300dcd092b980086b9b9c diff --git a/conf/.gitignore b/conf/.gitignore new file mode 100644 index 0000000..143ff36 --- /dev/null +++ b/conf/.gitignore @@ -0,0 +1,2 @@ +local.conf +sanity_info diff --git a/conf/bblayers.conf.mlinux b/conf/bblayers.conf.mlinux new file mode 100644 index 0000000..fb68424 --- /dev/null +++ b/conf/bblayers.conf.mlinux @@ -0,0 +1,50 @@ +LCONF_VERSION = "6" +TOPDIR := "${@os.path.dirname(os.path.dirname(d.getVar('FILE', True)))}" +BBPATH = "__OEROOT__" +BBFILES ?= "" +MACHINE ?= "mtcdt" + +# MACHINE SPECIFIC LAYER CONFIGURATIONS +MTBSP_AT91_LIST = "['mtcdt', 'mtcap', 'mtr', 'mtrv1', 'mtre']" +MTBSP_NXP_LIST = "['mtcap3']" + +MTBSP = "${@'mtbsp-at91' if '${MACHINE}' in ${MTBSP_AT91_LIST} else \ + 'mtbsp-nxp' if '${MACHINE}' in ${MTBSP_NXP_LIST} else ''}" + +OVERRIDES .= ":${MTBSP}" + +# Atmel AT91 +BSPPATH_mtbsp-at91 = "" +BSPLAYER_mtbsp-at91= "__OEROOT__/layers/meta-multitech-atmel \ + __OEROOT__/layers/meta-mlinux-atmel" + +BSPPATH_mtbsp-nxp = "__OEROOT__/layers/meta-multitech-nxp/meta-freescale" +BSPLAYER_mtbsp-nxp= "__OEROOT__/layers/meta-multitech-nxp \ + __OEROOT__/layers/meta-multitech-nxp/meta-freescale \ + __OEROOT__/layers/meta-mlinux-nxp \ + __OEROOT__/layers/meta-security/meta-tpm" + +# Append the BSP configuration +BBPATH =. "${@'${BSPPATH_${MTBSP}}:' if '${BSPPATH_${MTBSP}}' else ''}" + +# No virtaulization and swupdate on at91 (mask bbappends and images) +BBMASK_mtbsp-at91 += "wl18xx-fw tpm2-tss-engine" + +BBLAYERS = " \ + __OEROOT__/layers/user-layer \ + ${BSPLAYER_${MTBSP}} \ + __OEROOT__/layers/meta-mlinux \ + __OEROOT__/layers/meta-multitech \ + __OEROOT__/layers/meta-mono \ + __OEROOT__/layers/meta-java \ + __OEROOT__/layers/meta-python2 \ + __OEROOT__/layers/meta-swupdate \ + __OEROOT__/layers/meta-nodejs \ + __OEROOT__/layers/meta-openembedded/meta-oe \ + __OEROOT__/layers/meta-openembedded/meta-python \ + __OEROOT__/layers/meta-openembedded/meta-networking \ + __OEROOT__/layers/meta-openembedded/meta-webserver \ + __OEROOT__/layers/meta-openembedded/meta-multimedia \ + __OEROOT__/layers/meta-openembedded/meta-filesystems \ + __OEROOT__/layers/openembedded-core/meta \ + " diff --git a/conf/bblayers.conf.sample b/conf/bblayers.conf.sample new file mode 100644 index 0000000..a8e65be --- /dev/null +++ b/conf/bblayers.conf.sample @@ -0,0 +1,22 @@ +LCONF_VERSION = "6" +TOPDIR := "${@os.path.dirname(os.path.dirname(d.getVar('FILE', True)))}" +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS = " \ + ${TOPDIR}/layers/user-layer \ + ${TOPDIR}/layers/meta-mlinux \ + ${TOPDIR}/layers/meta-mlinux-atmel \ + ${TOPDIR}/layers/meta-multitech \ + ${TOPDIR}/layers/meta-multitech-atmel \ + ${TOPDIR}/layers/meta-mono \ + ${TOPDIR}/layers/meta-nodejs \ + ${TOPDIR}/layers/meta-java \ + ${TOPDIR}/layers/meta-openembedded/meta-oe \ + ${TOPDIR}/layers/meta-openembedded/meta-perl \ + ${TOPDIR}/layers/meta-openembedded/meta-networking \ + ${TOPDIR}/layers/meta-openembedded/meta-webserver \ + ${TOPDIR}/layers/meta-openembedded/meta-multimedia \ + ${TOPDIR}/layers/meta-openembedded/meta-filesystems \ + ${TOPDIR}/layers/meta-openembedded/toolchain-layer \ + ${TOPDIR}/layers/openembedded-core/meta \ + " diff --git a/conf/local.conf.sample b/conf/local.conf.sample new file mode 100644 index 0000000..f798315 --- /dev/null +++ b/conf/local.conf.sample @@ -0,0 +1,51 @@ +# CONF_VERSION is increased each time build/conf/ changes incompatibly +CONF_VERSION = "2" + +# Use MultiTech mLinux distribution +DISTRO = "mlinux" +MACHINE ?= "mtcdt" + +# Where to store downloaded sources +DL_DIR = "${TOPDIR}/downloads" + +# Where to save shared state +SSTATE_DIR = "${TOPDIR}/build/sstate-cache" +# bitbake cache location +PERSISTENT_DIR = "${TOPDIR}/build/cache" +# build output +TMPDIR = "${TOPDIR}/build/tmp" + +# Which files do we want to parse: +BBFILES ?= "${TOPDIR}/layers/openembedded-core/meta/recipes-*/*/*.bb" + +# Go through the Firewall +#HTTP_PROXY = "http://:/" + +# Uncomment this to remove unpacked source and intermediate work +# after successfully building a package. +# Saves a LOT of disk space, but leaving work around is useful for debugging. +INHERIT += "rm_work" + +# skip parsing of masked files +BBMASK = "" + +# Make use of SMP: +# PARALLEL_MAKE specifies how many concurrent compiler threads are spawned per bitbake process +# BB_NUMBER_THREADS specifies how many concurrent bitbake tasks will be run +BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()*2}" +PARALLEL_MAKE ?= "-j ${@oe.utils.cpu_count()*2}" + +# Don't generate the mirror tarball for SCM repos, the snapshot is enough +BB_GENERATE_MIRROR_TARBALLS = "0" + +# Disable build time patch resolution. This would lauch a devshell +# and wait for manual intervention. We disable it. +PATCHRESOLVE = "noop" + +# enable local PR server +PRSERV_HOST = "localhost:0" + +# enable buildhistory for images +INHERIT += "buildhistory" +BUILDHISTORY_COMMIT = "0" +BUILDHISTORY_FEATURES = "image" diff --git a/install-deps/install-centos-rhel-deps.sh b/install-deps/install-centos-rhel-deps.sh new file mode 100755 index 0000000..0f0e6d9 --- /dev/null +++ b/install-deps/install-centos-rhel-deps.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +yum install gawk make wget tar bzip2 gzip python unzip perl patch \ +diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath SDL-devel xterm diff --git a/install-deps/install-debian-ubuntu-deps.sh b/install-deps/install-debian-ubuntu-deps.sh new file mode 100755 index 0000000..6088db4 --- /dev/null +++ b/install-deps/install-debian-ubuntu-deps.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +apt-get install gawk wget git-core diffstat unzip texinfo gcc-multilib \ +build-essential chrpath libsdl1.2-dev xterm diff --git a/install-deps/install-fedora-deps.sh b/install-deps/install-fedora-deps.sh new file mode 100755 index 0000000..f1c34d4 --- /dev/null +++ b/install-deps/install-fedora-deps.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +yum install gawk make wget tar bzip2 gzip python unzip perl patch \ +diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath \ +ccache perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue SDL-devel xterm diff --git a/install-deps/install-opensuse-deps.sh b/install-deps/install-opensuse-deps.sh new file mode 100755 index 0000000..7f3c193 --- /dev/null +++ b/install-deps/install-opensuse-deps.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +zypper install python gcc gcc-c++ git chrpath make wget python-xml \ +diffstat texinfo python-curses patch libSDL-devel xterm diff --git a/layers/meta-java b/layers/meta-java new file mode 160000 +Subproject 6e84638d77ac921aac46649095bca5ddbde94d2 diff --git a/layers/meta-mlinux b/layers/meta-mlinux new file mode 160000 +Subproject 6c8886126e74b4e3dbd33bd075ec5f446ae9550 diff --git a/layers/meta-mlinux-atmel b/layers/meta-mlinux-atmel new file mode 160000 +Subproject 8c79c938c68f28385b0b47146ac95277f742974 diff --git a/layers/meta-mono b/layers/meta-mono new file mode 160000 +Subproject 0c5afc77a4ec61326996f006acac6f025f56a8b diff --git a/layers/meta-multitech b/layers/meta-multitech new file mode 160000 +Subproject cec6945d9087a9b66c5107c66f1029eca3f6236 diff --git a/layers/meta-multitech-atmel b/layers/meta-multitech-atmel new file mode 160000 +Subproject 58a6156b61364d8effe0876afc4ab9e0ce62c39 diff --git a/layers/meta-nodejs b/layers/meta-nodejs new file mode 160000 +Subproject eec531e97a17bfd406f3bf76dee4057dcf5286a diff --git a/layers/meta-openembedded b/layers/meta-openembedded new file mode 160000 +Subproject 85f8047c71047d93cf79a954142157f85079968 diff --git a/layers/meta-python2 b/layers/meta-python2 new file mode 160000 +Subproject b901080cf57d9a7f5476ab4d96e56c30db8170a diff --git a/layers/meta-security b/layers/meta-security new file mode 160000 +Subproject c74cc97641fd93e0e7a4383255e9a0ab3deaf9d diff --git a/layers/meta-swupdate b/layers/meta-swupdate new file mode 160000 +Subproject be283cb6be560dee5e16451ad5f31ec9db2b1db diff --git a/layers/meta-virtualization b/layers/meta-virtualization new file mode 160000 +Subproject c5f61e547b90aa8058cf816f00902afed9c96f7 diff --git a/layers/openembedded-core b/layers/openembedded-core new file mode 160000 +Subproject 9ae339ace9274be71bfd3b5e5da64dceac9fa96 diff --git a/oe-init-build-env b/oe-init-build-env new file mode 100755 index 0000000..525c23f --- /dev/null +++ b/oe-init-build-env @@ -0,0 +1,77 @@ +#!/bin/sh + +# OE Build Environment Setup Script +# +# Copyright (C) 2006-2011 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# +# Normally this is called as '. ./oe-init-build-env' +# + +if [ -n "$BASH_SOURCE" ]; then + THIS_SCRIPT=$BASH_SOURCE +elif [ -n "$ZSH_NAME" ]; then + THIS_SCRIPT=$0 +else + THIS_SCRIPT="$(pwd)/oe-init-build-env" +fi +if [ -n "$BBSERVER" ]; then + unset BBSERVER +fi + +if [ -z "$ZSH_NAME" ] && [ "$0" = "$THIS_SCRIPT" ]; then + echo "Error: This script needs to be sourced. Please run as '. $THIS_SCRIPT'" + exit 1 +fi + +if [ -z "$OEROOT" ]; then + OEROOT=$(dirname "$THIS_SCRIPT") + OEROOT=$(readlink -f "$OEROOT") +fi +unset THIS_SCRIPT + +export OEROOT +. $OEROOT/scripts/oe-buildenv-internal && + TEMPLATECONF="$TEMPLATECONF" $OEROOT/scripts/oe-setup-builddir || { + unset OEROOT + return 1 +} + +# Shutdown any bitbake server if the BBSERVER variable is not set +if [ -z "$BBSERVER" ] && [ -f $OEROOT/bitbake.lock ]; then + grep ":" $OEROOT/bitbake.lock > /dev/null && BBSERVER=$(cat $OEROOT/bitbake.lock) bitbake --status-only + if [ $? = 0 ]; then + echo "Shutting down bitbake memory resident server with bitbake -m" + BBSERVER=$(cat $OEROOT/bitbake.lock) bitbake -m + fi +fi + +export BBPATH="$OEROOT" + +# define if you wish to create soft links to the image in the tftpboot dir +#export TFTPBOOT_DIR=/tftpboot + +declare -a WlAdditions=("OEROOT") + +for entry in ${WlAdditions[@]}; do + if [ "$(echo "$BB_ENV_EXTRAWHITE" | sed -re 's/[ \t]+/\n/g' | egrep ''^$entry'')" != "$entry" ] ; then + echo $BB_ENV_EXTRAWHITE needs correction + BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE $entry" + export BB_ENV_EXTRAWHITE + fi +done + diff --git a/oe-init-build-env-memres b/oe-init-build-env-memres new file mode 100755 index 0000000..f3db2b6 --- /dev/null +++ b/oe-init-build-env-memres @@ -0,0 +1,94 @@ +#!/bin/sh + +# OE Build Environment Setup Script +# +# Copyright (C) 2006-2011 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# +# Normally this is called as '. ./oe-init-build-env-memres <portnumber>' +# +# This works in most shells (not dash), but not all of them pass the arguments +# when being sourced. To workaround the shell limitation use "set <portnumber>" +# prior to sourcing this script. +# +if [ -z "$1" ]; then + echo "No port specified, using dynamically selected port" + port=-1 +else + port=$1 + shift +fi + +if [ -n "$BASH_SOURCE" ]; then + THIS_SCRIPT=$BASH_SOURCE +elif [ -n "$ZSH_NAME" ]; then + THIS_SCRIPT=$0 +else + THIS_SCRIPT="$(pwd)/oe-init-build-env" +fi +if [ -n "$BBSERVER" ]; then + unset BBSERVER +fi + +if [ -z "$ZSH_NAME" ] && [ "$0" = "$THIS_SCRIPT" ]; then + echo "Error: This script needs to be sourced. Please run as '. $THIS_SCRIPT'" + exit 1 +fi + +if [ -z "$OEROOT" ]; then + OEROOT=$(dirname "$THIS_SCRIPT") + OEROOT=$(readlink -f "$OEROOT") +fi +unset THIS_SCRIPT + +export OEROOT +. $OEROOT/scripts/oe-buildenv-internal && + TEMPLATECONF="$TEMPLATECONF" $OEROOT/scripts/oe-setup-builddir || { + unset OEROOT + return 1 +} + +res=1 +if [ -e $OEROOT/bitbake.lock ] && grep : $OEROOT/bitbake.lock > /dev/null; then + BBSERVER=$(cat $OEROOT/bitbake.lock) bitbake --status-only + res=$? +fi + +if [ $res != 0 ]; then + bitbake --server-only -t xmlrpc -B localhost:$port +fi + +if [ $port = -1 ]; then + export BBSERVER=localhost:-1 + echo "Bitbake server started on demand as needed, use bitbake -m to shut it down" +else + export BBSERVER=$(cat $OEROOT/bitbake.lock) + + if [ $res = 0 ]; then + echo "Using existing bitbake server at: $BBSERVER, use bitbake -m to shut it down" + else + echo "Bitbake server started at: $BBSERVER, use bitbake -m to shut it down" + fi +fi +unset port res + +export BBPATH="$OEROOT" + +# define if you wish to create soft links to the image in the tftpboot dir +#export TFTPBOOT_DIR=/tftpboot + +unset OEROOT diff --git a/patches/bitbake_checksumwarningsuppress.patch b/patches/bitbake_checksumwarningsuppress.patch new file mode 100644 index 0000000..438a6f2 --- /dev/null +++ b/patches/bitbake_checksumwarningsuppress.patch @@ -0,0 +1,18 @@ +diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py +index dc99914c..f622fc7b 100644 +--- a/bitbake/lib/bb/fetch2/__init__.py ++++ b/bitbake/lib/bb/fetch2/__init__.py +@@ -1188,11 +1188,12 @@ def get_checksum_file_list(d): + + if ud and isinstance(ud.method, local.Local): + paths = ud.method.localpaths(ud, d) ++ strict = d.getVar("BB_STRICT_CHECKSUM", True) or "0" + for f in paths: + pth = ud.decodedurl + if '*' in pth: + f = os.path.join(os.path.abspath(f), pth) +- if f.startswith(dl_dir): ++ if f.startswith(dl_dir) and (strict == "1"): + # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else + if os.path.exists(f): + bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f))) diff --git a/patches/dont_use_network_on_unpack_of_tag.patch b/patches/dont_use_network_on_unpack_of_tag.patch new file mode 100644 index 0000000..6946535 --- /dev/null +++ b/patches/dont_use_network_on_unpack_of_tag.patch @@ -0,0 +1,95 @@ +diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py +index 112b833f..8c6d04d9 100644 +--- a/bitbake/lib/bb/fetch2/git.py ++++ b/bitbake/lib/bb/fetch2/git.py +@@ -129,6 +129,19 @@ class Git(FetchMethod): + def supports_checksum(self, urldata): + return False + ++ # git HASH verification code (is the REV a hash or not) ++ def bad_git_hash(self, revision): ++ if not revision or len(revision) != 40 or (False in [c in "abcdef0123456789" for c in revision]): ++ if not revision: ++ bb.note("bad_git_hash: SRCREV is not a defined string") ++ elif len(revision) != 40: ++ bb.debug(2,"bad_git_hash: revision len is not 40: " + str(len(revision))) ++ bb.debug(2,"bad_git_hash: revision dump: %s" % ":".join("{:02x}".format(ord(c)) for c in revision)) ++ else: ++ bb.debug(2,"bad_git_hash: revision bad char: %s" % ":".join("{:02x}".format(ord(c)) for c in revision)) ++ return True ++ return False ++ + def urldata_init(self, ud, d): + """ + init git specific variable within url data +@@ -229,16 +242,52 @@ class Git(FetchMethod): + + ud.setup_revisions(d) + ++ # EARLY_GITSRCNAME start ++ gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) ++ if gitsrcname.startswith('.'): ++ gitsrcname = gitsrcname[1:] ++ # EARLY_GITSRCNAME end ++ ++ # EARLY_GITDIR start ++ dl_dir = d.getVar("DL_DIR") ++ gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") ++ ud.clonedir = os.path.join(gitdir, gitsrcname) ++ ud.localfile = ud.clonedir ++ # EARLY_GITDIR end ++ + for name in ud.names: + # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one +- if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): ++ # First try the local repository ++ if self.bad_git_hash(ud.revisions[name]): + if ud.revisions[name]: + ud.unresolvedrev[name] = ud.revisions[name] +- ud.revisions[name] = self.latest_revision(ud, d, name) +- +- gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) +- if gitsrcname.startswith('.'): +- gitsrcname = gitsrcname[1:] ++ if len(ud.revisions[name]) != 0: ++ # Assume this is a tag, and retrieve hash from local source, ++ # if possible. Deviation from openembedded, which goes to ++ # remote source for tags. ++ cmd = "%s rev-parse %s^{}" % (ud.basecmd, ud.revisions[name]) ++ savedrev = ud.revisions[name] ++ if os.path.isdir(ud.localfile): ++ try: ++ # Validate local git repository ++ cmd = "%s --bare fsck --no-full" % (ud.basecmd) ++ runfetchcmd(cmd,d,workdir=ud.localfile) ++ # Valid repository, so see if we can get the hash ++ cmd = "%s rev-parse %s^{}" % (ud.basecmd, ud.revisions[name]) ++ ud.revisions[name] = runfetchcmd(cmd,d,workdir=ud.localfile).rstrip() ++ except bb.fetch.FetchError: ++ bb.note("do_fetch: removing invalid git file system: " + ud.localfile) ++ cmd = "rm -rf " + ud.localfile + ";rm -f " + ud.localfile + ".done" ++ ud.revisions[name] = savedrev ++ ++ if self.bad_git_hash(ud.revisions[name]): ++ ud.revisions[name] = self.latest_revision(ud, d, name) ++ ++ bb.debug(2,"urldata_init: ud.revisions[name] " + ud.revisions[name]) ++ bb.debug(2,"urldata_init: ud.host " + ud.host) ++ bb.debug(2,"urldata_init: ud.path " + ud.path) ++ ++ # gitsrcname code purposely moved to EARLY_GITSRCNAME + + # for rebaseable git repo, it is necessary to keep mirror tar ball + # per revision, so that even the revision disappears from the +@@ -248,10 +297,7 @@ class Git(FetchMethod): + for name in ud.names: + gitsrcname = gitsrcname + '_' + ud.revisions[name] + +- dl_dir = d.getVar("DL_DIR") +- gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") +- ud.clonedir = os.path.join(gitdir, gitsrcname) +- ud.localfile = ud.clonedir ++ # gitdir/clonedir/localfile purposely moved to EARLY_GITDIR + + mirrortarball = 'git2_%s.tar.gz' % gitsrcname + ud.fullmirror = os.path.join(dl_dir, mirrortarball) diff --git a/patches/kernel-revision-ext-mod.patch b/patches/kernel-revision-ext-mod.patch new file mode 100644 index 0000000..c7da73c --- /dev/null +++ b/patches/kernel-revision-ext-mod.patch @@ -0,0 +1,25 @@ +diff -Naru orig/layers/openembedded-core/meta/classes/kernel-module-split.bbclass new/layers/openembedded-core/meta/classes/kernel-module-split.bbclass +--- orig/layers/openembedded-core/meta/classes/kernel-module-split.bbclass 2020-07-13 17:38:32.148762374 -0500 ++++ new/layers/openembedded-core/meta/classes/kernel-module-split.bbclass 2020-07-13 17:36:27.740764540 -0500 +@@ -33,7 +33,7 @@ + KERNEL_MODULES_META_PACKAGE ?= "${@ d.getVar("KERNEL_PACKAGE_NAME") or "kernel" }-modules" + + KERNEL_MODULE_PACKAGE_PREFIX ?= "" +-KERNEL_MODULE_PACKAGE_SUFFIX ?= "-${KERNEL_VERSION}" ++KERNEL_MODULE_PACKAGE_SUFFIX ?= "-${KERNEL_VERSION}${KERNEL_REVISION}" + KERNEL_MODULE_PROVIDE_VIRTUAL ?= "1" + + python split_kernel_module_packages () { +diff -Naru orig/layers/openembedded-core/meta/classes/module-base.bbclass new/layers/openembedded-core/meta/classes/module-base.bbclass +--- orig/layers/openembedded-core/meta/classes/module-base.bbclass 2020-07-13 17:38:20.708762573 -0500 ++++ new/layers/openembedded-core/meta/classes/module-base.bbclass 2020-07-13 17:36:52.756764104 -0500 +@@ -16,6 +16,9 @@ + export KERNEL_VERSION = "${@oe.utils.read_file('${STAGING_KERNEL_BUILDDIR}/kernel-abiversion')}" + KERNEL_OBJECT_SUFFIX = ".ko" + ++# Kernel revision ++export KERNEL_REVISION = "${@oe.utils.read_file('${STAGING_KERNEL_BUILDDIR}/mlinux_pr')}" ++ + # kernel modules are generally machine specific + PACKAGE_ARCH = "${MACHINE_ARCH}" + diff --git a/patches/meta-openembedded-github-https.patch b/patches/meta-openembedded-github-https.patch new file mode 100644 index 0000000..9a49b85 --- /dev/null +++ b/patches/meta-openembedded-github-https.patch @@ -0,0 +1,171 @@ +github no longer allows clear git protocol, so switch to https +=============================================================== +diff --git a/layers/meta-openembedded/meta-filesystems/recipes-utils/ufs-utils/ufs-utils_git.bb b/layers/meta-openembedded/meta-filesystems/recipes-utils/ufs-utils/ufs-utils_git.bb +index 23583650b..ed003ee7b 100644 +--- a/layers/meta-openembedded/meta-filesystems/recipes-utils/ufs-utils/ufs-utils_git.bb ++++ b/layers/meta-openembedded/meta-filesystems/recipes-utils/ufs-utils/ufs-utils_git.bb +@@ -8,7 +8,7 @@ BRANCH ?= "dev" + + SRCREV = "a3cf93b66f4606a46354cf884d24aa966661f848" + +-SRC_URI = "git://github.com/westerndigitalcorporation/ufs-utils.git;protocol=git;branch=${BRANCH} \ ++SRC_URI = "git://github.com/westerndigitalcorporation/ufs-utils.git;protocol=https;branch=${BRANCH} \ + file://0001-Replace-u_intXX_t-with-kernel-typedefs.patch \ + " + +diff --git a/layers/meta-openembedded/meta-multimedia/recipes-multimedia/fdk-aac/fdk-aac_2.0.1.bb b/layers/meta-openembedded/meta-multimedia/recipes-multimedia/fdk-aac/fdk-aac_2.0.1.bb +index d7911681c..c499119c6 100644 +--- a/layers/meta-openembedded/meta-multimedia/recipes-multimedia/fdk-aac/fdk-aac_2.0.1.bb ++++ b/layers/meta-openembedded/meta-multimedia/recipes-multimedia/fdk-aac/fdk-aac_2.0.1.bb +@@ -11,7 +11,7 @@ LICENSE = "Fraunhofer_FDK_AAC_Codec_Library_for_Android" + LICENSE_FLAGS = "commercial" + LIC_FILES_CHKSUM = "file://NOTICE;md5=5985e1e12f4afa710d64ed7bfd291875" + +-SRC_URI = "git://github.com/mstorsjo/fdk-aac.git;protocol=git;branch=master" ++SRC_URI = "git://github.com/mstorsjo/fdk-aac.git;protocol=https;branch=master" + SRCREV = "d387d3b6ed79ff9a82c60440bdd86e6e5e324bec" + + S = "${WORKDIR}/git" +diff --git a/layers/meta-openembedded/meta-networking/recipes-protocols/babeld/babeld_1.9.1.bb b/layers/meta-openembedded/meta-networking/recipes-protocols/babeld/babeld_1.9.1.bb +index 6dd15ad9f..46b50c26b 100644 +--- a/layers/meta-openembedded/meta-networking/recipes-protocols/babeld/babeld_1.9.1.bb ++++ b/layers/meta-openembedded/meta-networking/recipes-protocols/babeld/babeld_1.9.1.bb +@@ -12,7 +12,7 @@ SECTION = "net" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://LICENCE;md5=411a48ac3c2e9e0911b8dd9aed26f754" + +-SRC_URI = "git://github.com/jech/babeld.git;protocol=git" ++SRC_URI = "git://github.com/jech/babeld.git;protocol=https" + SRCREV = "0835d5d894ea016ab7b81562466cade2c51a12d4" + + UPSTREAM_CHECK_GITTAGREGEX = "babeld-(?P<pver>\d+(\.\d+)+)" +diff --git a/layers/meta-openembedded/meta-networking/recipes-support/smcroute/smcroute_2.4.4.bb b/layers/meta-openembedded/meta-networking/recipes-support/smcroute/smcroute_2.4.4.bb +index 0b63f79ac..d8a1f6140 100644 +--- a/layers/meta-openembedded/meta-networking/recipes-support/smcroute/smcroute_2.4.4.bb ++++ b/layers/meta-openembedded/meta-networking/recipes-support/smcroute/smcroute_2.4.4.bb +@@ -6,7 +6,7 @@ LICENSE = "GPLv2+" + LIC_FILES_CHKSUM = "file://COPYING;md5=751419260aa954499f7abaabaa882bbe" + + SRCREV = "a8e5847e5f7e411be424f9b52a6cdf9d2ed4aeb5" +-SRC_URI = "git://github.com/troglobit/smcroute.git;branch=master;protocol=git" ++SRC_URI = "git://github.com/troglobit/smcroute.git;branch=master;protocol=https" + + S = "${WORKDIR}/git" + +diff --git a/layers/meta-openembedded/meta-oe/recipes-core/sdbus-c++/sdbus-c++-libsystemd_243.bb b/layers/meta-openembedded/meta-oe/recipes-core/sdbus-c++/sdbus-c++-libsystemd_243.bb +index c8e81a412..f0e928d0d 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-core/sdbus-c++/sdbus-c++-libsystemd_243.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-core/sdbus-c++/sdbus-c++-libsystemd_243.bb +@@ -12,7 +12,7 @@ DEPENDS += "gperf-native gettext-native util-linux libcap" + + SRCREV = "efb536d0cbe2e58f80e501d19999928c75e08f6a" + SRCBRANCH = "v243-stable" +-SRC_URI = "git://github.com/systemd/systemd-stable.git;protocol=git;branch=${SRCBRANCH}" ++SRC_URI = "git://github.com/systemd/systemd-stable.git;protocol=https;branch=${SRCBRANCH}" + + SRC_URI += "file://static-libsystemd-pkgconfig.patch" + +diff --git a/layers/meta-openembedded/meta-oe/recipes-extended/hiredis/hiredis_0.14.0.bb b/layers/meta-openembedded/meta-oe/recipes-extended/hiredis/hiredis_0.14.0.bb +index 29f8de8d2..b2f38728d 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-extended/hiredis/hiredis_0.14.0.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-extended/hiredis/hiredis_0.14.0.bb +@@ -6,7 +6,7 @@ DEPENDS = "redis" + + LIC_FILES_CHKSUM = "file://COPYING;md5=d84d659a35c666d23233e54503aaea51" + SRCREV = "685030652cd98c5414ce554ff5b356dfe8437870" +-SRC_URI = "git://github.com/redis/hiredis;protocol=git \ ++SRC_URI = "git://github.com/redis/hiredis;protocol=https \ + file://0001-Makefile-remove-hardcoding-of-CC.patch" + + S = "${WORKDIR}/git" +diff --git a/layers/meta-openembedded/meta-oe/recipes-extended/socketcan/can-utils_git.bb b/layers/meta-openembedded/meta-oe/recipes-extended/socketcan/can-utils_git.bb +index 519368817..5015ac53b 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-extended/socketcan/can-utils_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-extended/socketcan/can-utils_git.bb +@@ -4,7 +4,7 @@ LIC_FILES_CHKSUM = "file://include/linux/can.h;endline=44;md5=a9e1169c6c9a114a61 + + DEPENDS = "libsocketcan" + +-SRC_URI = "git://github.com/linux-can/${BPN}.git;protocol=git" ++SRC_URI = "git://github.com/linux-can/${BPN}.git;protocol=https" + + SRCREV = "da65fdfe0d1986625ee00af0b56ae17ec132e700" + +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/gpm/gpm_git.bb b/layers/meta-openembedded/meta-oe/recipes-support/gpm/gpm_git.bb +index 3800d147f..37631acae 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/gpm/gpm_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/gpm/gpm_git.bb +@@ -13,7 +13,7 @@ SRCREV = "1fd19417b8a4dd9945347e98dfa97e4cfd798d77" + + DEPENDS = "ncurses bison-native" + +-SRC_URI = "git://github.com/telmich/gpm;protocol=git \ ++SRC_URI = "git://github.com/telmich/gpm;protocol=https \ + file://init \ + file://gpm.service.in \ + file://0001-Use-sigemptyset-API-instead-of-__sigemptyset.patch \ +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/pidgin/funyahoo-plusplus_git.bb b/layers/meta-openembedded/meta-oe/recipes-support/pidgin/funyahoo-plusplus_git.bb +index 3a437659e..0e3e5ff73 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/pidgin/funyahoo-plusplus_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/pidgin/funyahoo-plusplus_git.bb +@@ -7,7 +7,7 @@ DEPENDS = "pidgin json-glib glib-2.0" + + inherit pkgconfig + +-SRC_URI = "git://github.com/EionRobb/funyahoo-plusplus;branch=master;protocol=git" ++SRC_URI = "git://github.com/EionRobb/funyahoo-plusplus;branch=master;protocol=https" + SRCREV = "fbbd9c591100aa00a0487738ec7b6acd3d924b3f" + + S = "${WORKDIR}/git" +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/pidgin/purple-skypeweb_git.bb b/layers/meta-openembedded/meta-oe/recipes-support/pidgin/purple-skypeweb_git.bb +index 092e6059b..854920d2e 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/pidgin/purple-skypeweb_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/pidgin/purple-skypeweb_git.bb +@@ -7,7 +7,7 @@ DEPENDS = "pidgin json-glib glib-2.0 zlib" + + inherit pkgconfig + +-SRC_URI = "git://github.com/EionRobb/skype4pidgin;branch=master;protocol=git" ++SRC_URI = "git://github.com/EionRobb/skype4pidgin;branch=master;protocol=https" + SRCREV = "14f1b69b6292bbdc98cca484b050ec8359394c4e" + + S = "${WORKDIR}/git" +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/rsnapshot/rsnapshot_git.bb b/layers/meta-openembedded/meta-oe/recipes-support/rsnapshot/rsnapshot_git.bb +index 33f5dccca..6fe8aa76f 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/rsnapshot/rsnapshot_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/rsnapshot/rsnapshot_git.bb +@@ -25,7 +25,7 @@ RDEPENDS_${PN} = "rsync \ + SRCREV = "a9e29850fc33c503c289e245c7bad350eed746d9" + PV = "1.4.3+git${SRCPV}" + +-SRC_URI = "git://github.com/DrHyde/${BPN};branch=master;protocol=git \ ++SRC_URI = "git://github.com/DrHyde/${BPN};branch=master;protocol=https \ + file://configure-fix-cmd_rsync.patch \ + " + +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/spdlog/spdlog_1.5.0.bb b/layers/meta-openembedded/meta-oe/recipes-support/spdlog/spdlog_1.5.0.bb +index 39629cce0..9294d1a70 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/spdlog/spdlog_1.5.0.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/spdlog/spdlog_1.5.0.bb +@@ -4,7 +4,7 @@ LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COREBASE}/meta/files/common-licenses/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + + SRCREV = "cf6f1dd01e660d5865d68bf5fa78f6376b89470a" +-SRC_URI = "git://github.com/gabime/spdlog.git;protocol=git;branch=v1.x;" ++SRC_URI = "git://github.com/gabime/spdlog.git;protocol=https;branch=v1.x;" + + S = "${WORKDIR}/git" + +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/spitools/spitools_git.bb b/layers/meta-openembedded/meta-oe/recipes-support/spitools/spitools_git.bb +index 625756873..31c87c96f 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/spitools/spitools_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/spitools/spitools_git.bb +@@ -10,7 +10,7 @@ SRCREV = "4a36a84f7df291ddaebd397aecf0c8515256a8e0" + + S = "${WORKDIR}/git" + +-SRC_URI = "git://github.com/cpb-/spi-tools.git;protocol=git" ++SRC_URI = "git://github.com/cpb-/spi-tools.git;protocol=https" + + + inherit autotools diff --git a/patches/no_net_check.patch b/patches/no_net_check.patch new file mode 100644 index 0000000..4c9e306 --- /dev/null +++ b/patches/no_net_check.patch @@ -0,0 +1,12 @@ +diff --git a/layers/openembedded-core/meta/classes/sanity.bbclass b/layers/openembedded-core/meta/classes/sanity.bbclass +index 2325ee2..5d2a8d4 100644 +--- a/layers/openembedded-core/meta/classes/sanity.bbclass ++++ b/layers/openembedded-core/meta/classes/sanity.bbclass +@@ -367,6 +367,7 @@ def check_connectivity(d): + # URI's to check can be set in the CONNECTIVITY_CHECK_URIS variable + # using the same syntax as for SRC_URI. If the variable is not set + # the check is skipped ++ return "" + test_uris = (d.getVar('CONNECTIVITY_CHECK_URIS') or "").split() + retval = "" + diff --git a/patches/openembedded-core-github-https.patch b/patches/openembedded-core-github-https.patch new file mode 100644 index 0000000..bbeea1d --- /dev/null +++ b/patches/openembedded-core-github-https.patch @@ -0,0 +1,14 @@ +github no longer allows clear git protocol, so switch to https +=============================================================== +diff --git a/layers/openembedded-core/meta/recipes-core/systemd/systemd.inc b/layers/openembedded-core/meta/recipes-core/systemd/systemd.inc +index 3165d13f03..8b5260bb0d 100644 +--- a/layers/openembedded-core/meta/recipes-core/systemd/systemd.inc ++++ b/layers/openembedded-core/meta/recipes-core/systemd/systemd.inc +@@ -16,6 +16,6 @@ LIC_FILES_CHKSUM = "file://LICENSE.GPL2;md5=751419260aa954499f7abaabaa882bbe \ + + SRCREV = "3ceaa81c61b654ebf562464d142675bd4d57d7b6" + SRCBRANCH = "v244-stable" +-SRC_URI = "git://github.com/systemd/systemd-stable.git;protocol=git;branch=${SRCBRANCH}" ++SRC_URI = "git://github.com/systemd/systemd-stable.git;protocol=https;branch=${SRCBRANCH}" + + S = "${WORKDIR}/git" diff --git a/patches/rename-mtr-conflicts-with-mtr-product.patch b/patches/rename-mtr-conflicts-with-mtr-product.patch new file mode 100644 index 0000000..1a07a37 --- /dev/null +++ b/patches/rename-mtr-conflicts-with-mtr-product.patch @@ -0,0 +1,58 @@ +diff -Naru orig/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr_0.93.bb new/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr_0.93.bb +--- orig/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr_0.93.bb 2021-07-14 14:05:59.000000000 -0500 ++++ new/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr_0.93.bb 1969-12-31 18:00:00.000000000 -0600 +@@ -1,25 +0,0 @@ +-SUMMARY = "Combined traceroute and ping utility" +-DESCRIPTION = "mtr combines the functionality of the 'traceroute' and 'ping' programs in a single network diagnostic tool." +-HOMEPAGE = "http://www.bitwizard.nl/mtr/" +-SECTION = "net" +-DEPENDS = "ncurses" +- +-LICENSE = "GPLv2" +-LIC_FILES_CHKSUM = "file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263 \ +- file://ui/mtr.c;beginline=5;endline=16;md5=00a894a39d53726a27386534d1c4e468" +- +-SRCREV = "304349bad86229aedbc62c07d5e98a8292967991" +-SRC_URI = "git://github.com/traviscross/mtr" +- +-S = "${WORKDIR}/git" +- +-inherit autotools pkgconfig +- +-EXTRA_OECONF = "--without-gtk" +- +-PACKAGECONFIG ??= "${@bb.utils.filter('DISTRO_FEATURES', 'ipv6', d)}" +-PACKAGECONFIG[ipv6] = "--enable-ipv6,--disable-ipv6," +- +-PACKAGES += "${PN}-bash-completions" +- +-FILES_${PN}-bash-completions = "${datadir}/bash-completion/" +diff -Naru orig/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr-conflicts-with-mtr-product_0.93.bb new/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr-conflicts-with-mtr-product_0.93.bb +--- orig/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr-conflicts-with-mtr-product_0.93.bb 1969-12-31 18:00:00.000000000 -0600 ++++ new/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr-conflicts-with-mtr-product_0.93.bb 2021-07-14 14:05:59.000000000 -0500 +@@ -0,0 +1,25 @@ ++SUMMARY = "Combined traceroute and ping utility" ++DESCRIPTION = "mtr combines the functionality of the 'traceroute' and 'ping' programs in a single network diagnostic tool." ++HOMEPAGE = "http://www.bitwizard.nl/mtr/" ++SECTION = "net" ++DEPENDS = "ncurses" ++ ++LICENSE = "GPLv2" ++LIC_FILES_CHKSUM = "file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263 \ ++ file://ui/mtr.c;beginline=5;endline=16;md5=00a894a39d53726a27386534d1c4e468" ++ ++SRCREV = "304349bad86229aedbc62c07d5e98a8292967991" ++SRC_URI = "git://github.com/traviscross/mtr" ++ ++S = "${WORKDIR}/git" ++ ++inherit autotools pkgconfig ++ ++EXTRA_OECONF = "--without-gtk" ++ ++PACKAGECONFIG ??= "${@bb.utils.filter('DISTRO_FEATURES', 'ipv6', d)}" ++PACKAGECONFIG[ipv6] = "--enable-ipv6,--disable-ipv6," ++ ++PACKAGES += "${PN}-bash-completions" ++ ++FILES_${PN}-bash-completions = "${datadir}/bash-completion/" diff --git a/scripts/README b/scripts/README new file mode 100644 index 0000000..1b8d127 --- /dev/null +++ b/scripts/README @@ -0,0 +1 @@ +This directory contains Various useful scripts for working with OE builds diff --git a/scripts/bitbake-prserv-tool b/scripts/bitbake-prserv-tool new file mode 100755 index 0000000..fa31b52 --- /dev/null +++ b/scripts/bitbake-prserv-tool @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +help () +{ + base=`basename $0` + echo -e "Usage: $base command" + echo "Avaliable commands:" + echo -e "\texport <file.conf>: export and lock down the AUTOPR values from the PR service into a file for release." + echo -e "\timport <file.conf>: import the AUTOPR values from the exported file into the PR service." +} + +clean_cache() +{ + s=`bitbake -e | grep ^CACHE= | cut -f2 -d\"` + if [ "x${s}" != "x" ]; then + rm -rf ${s} + fi +} + +do_export () +{ + file=$1 + [ "x${file}" == "x" ] && help && exit 1 + rm -f ${file} + + clean_cache + bitbake -R conf/prexport.conf -p + s=`bitbake -R conf/prexport.conf -e | grep ^PRSERV_DUMPFILE= | cut -f2 -d\"` + if [ "x${s}" != "x" ]; + then + [ -e $s ] && mv -f $s $file && echo "Exporting to file $file succeeded!" + return 0 + fi + echo "Exporting to file $file failed!" + return 1 +} + +do_import () +{ + file=$1 + [ "x${file}" == "x" ] && help && exit 1 + + clean_cache + bitbake -R conf/primport.conf -R $file -p + ret=$? + [ $ret -eq 0 ] && echo "Importing from file $file succeeded!" || echo "Importing from file $file failed!" + return $ret +} + +do_migrate_localcount () +{ + df=`bitbake -R conf/migrate_localcount.conf -e | \ + grep ^LOCALCOUNT_DUMPFILE= | cut -f2 -d\"` + if [ "x${df}" == "x" ]; + then + echo "LOCALCOUNT_DUMPFILE is not defined!" + return 1 + fi + + rm -rf $df + clean_cache + echo "Exporting LOCALCOUNT to AUTOINCs..." + bitbake -R conf/migrate_localcount.conf -p + [ ! $? -eq 0 ] && echo "Exporting to file $df failed!" && exit 1 + + if [ -e $df ]; + then + echo "Exporting to file $df succeeded!" + else + echo "Exporting to file $df failed!" + exit 1 + fi + + echo "Importing generated AUTOINC entries..." + [ -e $df ] && do_import $df + + if [ ! $? -eq 0 ] + then + echo "Migration from LOCALCOUNT to AUTOINCs failed!" + return 1 + fi + + echo "Migration from LOCALCOUNT to AUTOINCs succeeded!" + return 0 +} + +[ $# -eq 0 ] && help && exit 1 + +case $2 in +*.conf|*.inc) + ;; +*) + echo ERROR: $2 must end with .conf or .inc! + exit 1 + ;; +esac + +case $1 in +export) + do_export $2 + ;; +import) + do_import $2 + ;; +migrate_localcount) + do_migrate_localcount + ;; +*) + help + exit 1 + ;; +esac diff --git a/scripts/bitbake-whatchanged b/scripts/bitbake-whatchanged new file mode 100755 index 0000000..af54d16 --- /dev/null +++ b/scripts/bitbake-whatchanged @@ -0,0 +1,339 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- + +# Copyright (c) 2013 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +from __future__ import print_function +import os +import sys +import getopt +import shutil +import re +import warnings +import subprocess +from optparse import OptionParser + +scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] + +import scriptpath + +# Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process +bitbakepath = scriptpath.add_bitbake_lib_path() +if not bitbakepath: + sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") + sys.exit(1) + +import bb.siggen +import bb.process + +# Match the stamp's filename +# group(1): PE_PV (may no PE) +# group(2): PR +# group(3): TASK +# group(4): HASH +stamp_re = re.compile("(?P<pv>.*)-(?P<pr>r\d+)\.(?P<task>do_\w+)\.(?P<hash>[^\.]*)") +sigdata_re = re.compile(".*\.sigdata\..*") + +def gen_dict(stamps): + """ + Generate the dict from the stamps dir. + The output dict format is: + {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}} + Where: + fake_f: pv + task + hash + path: the path to the stamp file + """ + # The member of the sub dict (A "path" will be appended below) + sub_mem = ("pv", "pr", "task") + d = {} + for dirpath, _, files in os.walk(stamps): + for f in files: + # The "bitbake -S" would generate ".sigdata", but no "_setscene". + fake_f = re.sub('_setscene.', '.', f) + fake_f = re.sub('.sigdata', '', fake_f) + subdict = {} + tmp = stamp_re.match(fake_f) + if tmp: + for i in sub_mem: + subdict[i] = tmp.group(i) + if len(subdict) != 0: + pn = os.path.basename(dirpath) + subdict['pn'] = pn + # The path will be used by os.stat() and bb.siggen + subdict['path'] = dirpath + "/" + f + fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash') + d[fake_f] = subdict + return d + +# Re-construct the dict +def recon_dict(dict_in): + """ + The output dict format is: + {pn_task: {pv: PV, pr: PR, path: PATH}} + """ + dict_out = {} + for k in dict_in.keys(): + subdict = {} + # The key + pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task')) + # If more than one stamps are found, use the latest one. + if pn_task in dict_out: + full_path_pre = dict_out.get(pn_task).get('path') + full_path_cur = dict_in.get(k).get('path') + if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime: + continue + subdict['pv'] = dict_in.get(k).get('pv') + subdict['pr'] = dict_in.get(k).get('pr') + subdict['path'] = dict_in.get(k).get('path') + dict_out[pn_task] = subdict + + return dict_out + +def split_pntask(s): + """ + Split the pn_task in to (pn, task) and return it + """ + tmp = re.match("(.*)_(do_.*)", s) + return (tmp.group(1), tmp.group(2)) + + +def print_added(d_new = None, d_old = None): + """ + Print the newly added tasks + """ + added = {} + for k in d_new.keys(): + if k not in d_old: + # Add the new one to added dict, and remove it from + # d_new, so the remaining ones are the changed ones + added[k] = d_new.get(k) + del(d_new[k]) + + if not added: + return 0 + + # Format the output, the dict format is: + # {pn: task1, task2 ...} + added_format = {} + counter = 0 + for k in added.keys(): + pn, task = split_pntask(k) + if pn in added_format: + # Append the value + added_format[pn] = "%s %s" % (added_format.get(pn), task) + else: + added_format[pn] = task + counter += 1 + print("=== Newly added tasks: (%s tasks)" % counter) + for k in added_format.keys(): + print(" %s: %s" % (k, added_format.get(k))) + + return counter + +def print_vrchanged(d_new = None, d_old = None, vr = None): + """ + Print the pv or pr changed tasks. + The arg "vr" is "pv" or "pr" + """ + pvchanged = {} + counter = 0 + for k in d_new.keys(): + if d_new.get(k).get(vr) != d_old.get(k).get(vr): + counter += 1 + pn, task = split_pntask(k) + if pn not in pvchanged: + # Format the output, we only print pn (no task) since + # all the tasks would be changed when pn or pr changed, + # the dict format is: + # {pn: pv/pr_old -> pv/pr_new} + pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr)) + del(d_new[k]) + + if not pvchanged: + return 0 + + print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter)) + for k in pvchanged.keys(): + print(" %s: %s" % (k, pvchanged.get(k))) + + return counter + +def print_depchanged(d_new = None, d_old = None, verbose = False): + """ + Print the dependency changes + """ + depchanged = {} + counter = 0 + for k in d_new.keys(): + counter += 1 + pn, task = split_pntask(k) + if (verbose): + full_path_old = d_old.get(k).get("path") + full_path_new = d_new.get(k).get("path") + # No counter since it is not ready here + if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new): + output = bb.siggen.compare_sigfiles(full_path_old, full_path_new) + if output: + print("\n=== The verbose changes of %s.%s:" % (pn, task)) + print('\n'.join(output)) + else: + # Format the output, the format is: + # {pn: task1, task2, ...} + if pn in depchanged: + depchanged[pn] = "%s %s" % (depchanged.get(pn), task) + else: + depchanged[pn] = task + + if len(depchanged) > 0: + print("\n=== Dependencies changed: (%s tasks)" % counter) + for k in depchanged.keys(): + print(" %s: %s" % (k, depchanged[k])) + + return counter + + +def main(): + """ + Print what will be done between the current and last builds: + 1) Run "STAMPS_DIR=<path> bitbake -S recipe" to re-generate the stamps + 2) Figure out what are newly added and changed, can't figure out + what are removed since we can't know the previous stamps + clearly, for example, if there are several builds, we can't know + which stamps the last build has used exactly. + 3) Use bb.siggen.compare_sigfiles to diff the old and new stamps + """ + + parser = OptionParser( + version = "1.0", + usage = """%prog [options] [package ...] +print what will be done between the current and last builds, for example: + + $ bitbake core-image-sato + # Edit the recipes + $ bitbake-whatchanged core-image-sato + +The changes will be printed" + +Note: + The amount of tasks is not accurate when the task is "do_build" since + it usually depends on other tasks. + The "nostamp" task is not included. +""" +) + parser.add_option("-v", "--verbose", help = "print the verbose changes", + action = "store_true", dest = "verbose") + + options, args = parser.parse_args(sys.argv) + + verbose = options.verbose + + if len(args) != 2: + parser.error("Incorrect number of arguments") + else: + recipe = args[1] + + # Get the STAMPS_DIR + print("Figuring out the STAMPS_DIR ...") + cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'" + try: + stampsdir, err = bb.process.run(cmdline) + except: + raise + if not stampsdir: + print("ERROR: No STAMPS_DIR found for '%s'" % recipe, file=sys.stderr) + return 2 + stampsdir = stampsdir.rstrip("\n") + if not os.path.isdir(stampsdir): + print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr) + return 2 + + # The new stamps dir + new_stampsdir = stampsdir + ".bbs" + if os.path.exists(new_stampsdir): + print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr) + return 2 + + try: + # Generate the new stamps dir + print("Generating the new stamps ... (need several minutes)") + cmdline = "STAMPS_DIR=%s bitbake -S none %s" % (new_stampsdir, recipe) + # FIXME + # The "bitbake -S" may fail, not fatal error, the stamps will still + # be generated, this might be a bug of "bitbake -S". + try: + bb.process.run(cmdline) + except Exception as exc: + print(exc) + + # The dict for the new and old stamps. + old_dict = gen_dict(stampsdir) + new_dict = gen_dict(new_stampsdir) + + # Remove the same one from both stamps. + cnt_unchanged = 0 + for k in new_dict.keys(): + if k in old_dict: + cnt_unchanged += 1 + del(new_dict[k]) + del(old_dict[k]) + + # Re-construct the dict to easily find out what is added or changed. + # The dict format is: + # {pn_task: {pv: PV, pr: PR, path: PATH}} + new_recon = recon_dict(new_dict) + old_recon = recon_dict(old_dict) + + del new_dict + del old_dict + + # Figure out what are changed, the new_recon would be changed + # by the print_xxx function. + # Newly added + cnt_added = print_added(new_recon, old_recon) + + # PV (including PE) and PR changed + # Let the bb.siggen handle them if verbose + cnt_rv = {} + if not verbose: + for i in ('pv', 'pr'): + cnt_rv[i] = print_vrchanged(new_recon, old_recon, i) + + # Dependencies changed (use bitbake-diffsigs) + cnt_dep = print_depchanged(new_recon, old_recon, verbose) + + total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep + + print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged)) + if verbose: + print("Newly added: %s\nDependencies changed: %s\n" % \ + (cnt_added, cnt_dep)) + else: + print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \ + (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep)) + except: + print("ERROR occurred!") + raise + finally: + # Remove the newly generated stamps dir + if os.path.exists(new_stampsdir): + print("Removing the newly generated stamps dir ...") + shutil.rmtree(new_stampsdir) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/buildhistory-collect-srcrevs b/scripts/buildhistory-collect-srcrevs new file mode 100755 index 0000000..f3eb76b --- /dev/null +++ b/scripts/buildhistory-collect-srcrevs @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# +# Collects the recorded SRCREV values from buildhistory and reports on them +# +# Copyright 2013 Intel Corporation +# Authored-by: Paul Eggleton <paul.eggleton@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import collections +import os +import sys +import optparse +import logging + +def logger_create(): + logger = logging.getLogger("buildhistory") + loggerhandler = logging.StreamHandler() + loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + logger.addHandler(loggerhandler) + logger.setLevel(logging.INFO) + return logger + +logger = logger_create() + +def main(): + parser = optparse.OptionParser( + description = "Collects the recorded SRCREV values from buildhistory and reports on them.", + usage = """ + %prog [options]""") + + parser.add_option("-a", "--report-all", + help = "Report all SRCREV values, not just ones where AUTOREV has been used", + action="store_true", dest="reportall") + parser.add_option("-f", "--forcevariable", + help = "Use forcevariable override for all output lines", + action="store_true", dest="forcevariable") + parser.add_option("-p", "--buildhistory-dir", + help = "Specify path to buildhistory directory (defaults to buildhistory/ under cwd)", + action="store", dest="buildhistory_dir", default='buildhistory/') + + options, args = parser.parse_args(sys.argv) + + if len(args) > 1: + sys.stderr.write('Invalid argument(s) specified: %s\n\n' % ' '.join(args[1:])) + parser.print_help() + sys.exit(1) + + if not os.path.exists(options.buildhistory_dir): + sys.stderr.write('Buildhistory directory "%s" does not exist\n\n' % options.buildhistory_dir) + parser.print_help() + sys.exit(1) + + if options.forcevariable: + forcevariable = '_forcevariable' + else: + forcevariable = '' + + all_srcrevs = collections.defaultdict(list) + for root, dirs, files in os.walk(options.buildhistory_dir): + if '.git' in dirs: + dirs.remove('.git') + for fn in files: + if fn == 'latest_srcrev': + curdir = os.path.basename(os.path.dirname(root)) + fullpath = os.path.join(root, fn) + pn = os.path.basename(root) + srcrev = None + orig_srcrev = None + orig_srcrevs = {} + srcrevs = {} + with open(fullpath) as f: + for line in f: + if '=' in line: + splitval = line.split('=') + value = splitval[1].strip('" \t\n\r') + if line.startswith('# SRCREV = '): + orig_srcrev = value + elif line.startswith('# SRCREV_'): + splitval = line.split('=') + name = splitval[0].split('_')[1].strip() + orig_srcrevs[name] = value + elif line.startswith('SRCREV ='): + srcrev = value + elif line.startswith('SRCREV_'): + name = splitval[0].split('_')[1].strip() + srcrevs[name] = value + if srcrev and (options.reportall or srcrev != orig_srcrev): + all_srcrevs[curdir].append((pn, None, srcrev)) + for name, value in srcrevs.items(): + orig = orig_srcrevs.get(name, orig_srcrev) + if options.reportall or value != orig: + all_srcrevs[curdir].append((pn, name, srcrev)) + + for curdir, srcrevs in sorted(all_srcrevs.iteritems()): + if srcrevs: + print('# %s' % curdir) + for pn, name, srcrev in srcrevs: + if name: + print('SRCREV_%s_pn-%s%s = "%s"' % (name, pn, forcevariable, srcrev)) + else: + print('SRCREV_pn-%s%s = "%s"' % (pn, forcevariable, srcrev)) + + +if __name__ == "__main__": + main() diff --git a/scripts/buildhistory-diff b/scripts/buildhistory-diff new file mode 100755 index 0000000..dfebcdd --- /dev/null +++ b/scripts/buildhistory-diff @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +# Report significant differences in the buildhistory repository since a specific revision +# +# Copyright (C) 2013 Intel Corporation +# Author: Paul Eggleton <paul.eggleton@linux.intel.com> + +import sys +import os +import optparse +from distutils.version import LooseVersion + +# Ensure PythonGit is installed (buildhistory_analysis needs it) +try: + import git +except ImportError: + print("Please install GitPython (python-git) 0.3.1 or later in order to use this script") + sys.exit(1) + +def main(): + parser = optparse.OptionParser( + description = "Reports significant differences in the buildhistory repository.", + usage = """ + %prog [options] [from-revision [to-revision]] +(if not specified, from-revision defaults to build-minus-1, and to-revision defaults to HEAD)""") + + parser.add_option("-p", "--buildhistory-dir", + help = "Specify path to buildhistory directory (defaults to buildhistory/ under cwd)", + action="store", dest="buildhistory_dir", default='buildhistory/') + parser.add_option("-v", "--report-version", + help = "Report changes in PKGE/PKGV/PKGR even when the values are still the default (PE/PV/PR)", + action="store_true", dest="report_ver", default=False) + parser.add_option("-a", "--report-all", + help = "Report all changes, not just the default significant ones", + action="store_true", dest="report_all", default=False) + + options, args = parser.parse_args(sys.argv) + + if len(args) > 3: + sys.stderr.write('Invalid argument(s) specified: %s\n\n' % ' '.join(args[3:])) + parser.print_help() + sys.exit(1) + + if LooseVersion(git.__version__) < '0.3.1': + sys.stderr.write("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script\n") + sys.exit(1) + + if not os.path.exists(options.buildhistory_dir): + sys.stderr.write('Buildhistory directory "%s" does not exist\n\n' % options.buildhistory_dir) + parser.print_help() + sys.exit(1) + + scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) + lib_path = scripts_path + '/lib' + sys.path = sys.path + [lib_path] + + import scriptpath + + # Set path to OE lib dir so we can import the buildhistory_analysis module + scriptpath.add_oe_lib_path() + # Set path to bitbake lib dir so the buildhistory_analysis module can load bb.utils + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") + sys.exit(1) + + import oe.buildhistory_analysis + + fromrev = 'build-minus-1' + torev = 'HEAD' + if len(args) > 1: + if len(args) == 2 and '..' in args[1]: + revs = args[1].split('..') + fromrev = revs[0] + if revs[1]: + torev = revs[1] + else: + fromrev = args[1] + if len(args) > 2: + torev = args[2] + + import gitdb + try: + changes = oe.buildhistory_analysis.process_changes(options.buildhistory_dir, fromrev, torev, options.report_all, options.report_ver) + except gitdb.exc.BadObject as e: + if len(args) == 1: + sys.stderr.write("Unable to find previous build revision in buildhistory repository\n\n") + parser.print_help() + else: + sys.stderr.write('Specified git revision "%s" is not valid\n' % e.args[0]) + sys.exit(1) + + for chg in changes: + print('%s' % chg) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/cleanup-workdir b/scripts/cleanup-workdir new file mode 100755 index 0000000..01ebd52 --- /dev/null +++ b/scripts/cleanup-workdir @@ -0,0 +1,198 @@ +#!/usr/bin/env python + +# Copyright (c) 2012 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import sys +import optparse +import re +import subprocess +import shutil + +pkg_cur_dirs = {} +obsolete_dirs = [] +parser = None + +def err_quit(msg): + print msg + parser.print_usage() + sys.exit(1) + +def parse_version(verstr): + elems = verstr.split(':') + epoch = elems[0] + if len(epoch) == 0: + return elems[1] + else: + return epoch + '_' + elems[1] + +def run_command(cmd): + pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + output = pipe.communicate()[0] + if pipe.returncode != 0: + print "Execute command '%s' failed." % cmd + sys.exit(1) + return output + +def get_cur_arch_dirs(workdir, arch_dirs): + pattern = workdir + '/(.*?)/' + + cmd = "bitbake -e | grep ^SDK_ARCH=" + output = run_command(cmd) + sdk_arch = output.split('"')[1] + + # select thest 5 packages to get the dirs of current arch + pkgs = ['hicolor-icon-theme', 'base-files', 'acl-native', 'binutils-crosssdk-' + sdk_arch, 'nativesdk-autoconf'] + + for pkg in pkgs: + cmd = "bitbake -e " + pkg + " | grep ^IMAGE_ROOTFS=" + output = run_command(cmd) + output = output.split('"')[1] + m = re.match(pattern, output) + arch_dirs.append(m.group(1)) + +def main(): + global parser + parser = optparse.OptionParser( + usage = """%prog + +%prog removes the obsolete packages' build directories in WORKDIR. +This script must be ran under BUILDDIR after source file \"oe-init-build-env\". + +Any file or directory under WORKDIR which is not created by Yocto +will be deleted. Be CAUTIOUS.""") + + options, args = parser.parse_args(sys.argv) + + builddir = run_command('echo $BUILDDIR').strip() + if len(builddir) == 0: + err_quit("Please source file \"oe-init-build-env\" first.\n") + + if os.getcwd() != builddir: + err_quit("Please run %s under: %s\n" % (os.path.basename(args[0]), builddir)) + + print 'Updating bitbake caches...' + cmd = "bitbake -s" + output = run_command(cmd) + + output = output.split('\n') + index = 0 + while len(output[index]) > 0: + index += 1 + alllines = output[index+1:] + + for line in alllines: + # empty again means end of the versions output + if len(line) == 0: + break + line = line.strip() + line = re.sub('\s+', ' ', line) + elems = line.split(' ') + if len(elems) == 2: + version = parse_version(elems[1]) + else: + version = parse_version(elems[2]) + pkg_cur_dirs[elems[0]] = version + + cmd = "bitbake -e" + output = run_command(cmd) + + tmpdir = None + image_rootfs = None + output = output.split('\n') + for line in output: + if tmpdir and image_rootfs: + break + + if not tmpdir: + m = re.match('TMPDIR="(.*)"', line) + if m: + tmpdir = m.group(1) + + if not image_rootfs: + m = re.match('IMAGE_ROOTFS="(.*)"', line) + if m: + image_rootfs = m.group(1) + + # won't fail just in case + if not tmpdir or not image_rootfs: + print "Can't get TMPDIR or IMAGE_ROOTFS." + return 1 + + pattern = tmpdir + '/(.*?)/(.*?)/' + m = re.match(pattern, image_rootfs) + if not m: + print "Can't get WORKDIR." + return 1 + + workdir = os.path.join(tmpdir, m.group(1)) + + # we only deal the dirs of current arch, total numbers of dirs are 6 + cur_arch_dirs = [m.group(2)] + get_cur_arch_dirs(workdir, cur_arch_dirs) + + for workroot, dirs, files in os.walk(workdir): + # For the files, they should NOT exist in WORKDIR. Remove them. + for f in files: + obsolete_dirs.append(os.path.join(workroot, f)) + + for d in dirs: + if d not in cur_arch_dirs: + continue + + for pkgroot, pkgdirs, filenames in os.walk(os.path.join(workroot, d)): + for f in filenames: + obsolete_dirs.append(os.path.join(pkgroot, f)) + + for pkgdir in sorted(pkgdirs): + if pkgdir not in pkg_cur_dirs: + obsolete_dirs.append(os.path.join(pkgroot, pkgdir)) + else: + for verroot, verdirs, verfiles in os.walk(os.path.join(pkgroot, pkgdir)): + for f in verfiles: + obsolete_dirs.append(os.path.join(pkgroot, f)) + for v in sorted(verdirs): + if v not in pkg_cur_dirs[pkgdir]: + obsolete_dirs.append(os.path.join(pkgroot, pkgdir, v)) + break + + # just process the top dir of every package under tmp/work/*/, + # then jump out of the above os.walk() + break + + # it is convenient to use os.walk() to get dirs and files at same time + # both of them have been dealed in the loop, so jump out + break + + for d in obsolete_dirs: + print "Deleting %s" % d + shutil.rmtree(d, True) + + if len(obsolete_dirs): + print '\nTotal %d items.' % len(obsolete_dirs) + else: + print '\nNo obsolete directory found under %s.' % workdir + + return 0 + +if __name__ == '__main__': + try: + ret = main() + except Exception: + ret = 2 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/scripts/combo-layer b/scripts/combo-layer new file mode 100755 index 0000000..9127041 --- /dev/null +++ b/scripts/combo-layer @@ -0,0 +1,924 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright 2011 Intel Corporation +# Authored-by: Yu Ke <ke.yu@intel.com> +# Paul Eggleton <paul.eggleton@intel.com> +# Richard Purdie <richard.purdie@intel.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import fnmatch +import os, sys +import optparse +import logging +import subprocess +import tempfile +import ConfigParser +import re +from collections import OrderedDict +from string import Template + +__version__ = "0.2.1" + +def logger_create(): + logger = logging.getLogger("") + loggerhandler = logging.StreamHandler() + loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S")) + logger.addHandler(loggerhandler) + logger.setLevel(logging.INFO) + return logger + +logger = logger_create() + +def get_current_branch(repodir=None): + try: + if not os.path.exists(os.path.join(repodir if repodir else '', ".git")): + # Repo not created yet (i.e. during init) so just assume master + return "master" + branchname = runcmd("git symbolic-ref HEAD 2>/dev/null", repodir).strip() + if branchname.startswith("refs/heads/"): + branchname = branchname[11:] + return branchname + except subprocess.CalledProcessError: + return "" + +class Configuration(object): + """ + Manages the configuration + + For an example config file, see combo-layer.conf.example + + """ + def __init__(self, options): + for key, val in options.__dict__.items(): + setattr(self, key, val) + + def readsection(parser, section, repo): + for (name, value) in parser.items(section): + if value.startswith("@"): + self.repos[repo][name] = eval(value.strip("@")) + else: + # Apply special type transformations for some properties. + # Type matches the RawConfigParser.get*() methods. + types = {'signoff': 'boolean', 'update': 'boolean'} + if name in types: + value = getattr(parser, 'get' + types[name])(section, name) + self.repos[repo][name] = value + + def readglobalsection(parser, section): + for (name, value) in parser.items(section): + if name == "commit_msg": + self.commit_msg_template = value + + logger.debug("Loading config file %s" % self.conffile) + self.parser = ConfigParser.ConfigParser() + with open(self.conffile) as f: + self.parser.readfp(f) + + # initialize default values + self.commit_msg_template = "Automatic commit to update last_revision" + + self.repos = {} + for repo in self.parser.sections(): + if repo == "combo-layer-settings": + # special handling for global settings + readglobalsection(self.parser, repo) + else: + self.repos[repo] = {} + readsection(self.parser, repo, repo) + + # Load local configuration, if available + self.localconffile = None + self.localparser = None + self.combobranch = None + if self.conffile.endswith('.conf'): + lcfile = self.conffile.replace('.conf', '-local.conf') + if os.path.exists(lcfile): + # Read combo layer branch + self.combobranch = get_current_branch() + logger.debug("Combo layer branch is %s" % self.combobranch) + + self.localconffile = lcfile + logger.debug("Loading local config file %s" % self.localconffile) + self.localparser = ConfigParser.ConfigParser() + with open(self.localconffile) as f: + self.localparser.readfp(f) + + for section in self.localparser.sections(): + if '|' in section: + sectionvals = section.split('|') + repo = sectionvals[0] + if sectionvals[1] != self.combobranch: + continue + else: + repo = section + if repo in self.repos: + readsection(self.localparser, section, repo) + + def update(self, repo, option, value, initmode=False): + # If the main config has the option already, that is what we + # are expected to modify. + if self.localparser and not self.parser.has_option(repo, option): + parser = self.localparser + section = "%s|%s" % (repo, self.combobranch) + conffile = self.localconffile + if initmode and not parser.has_section(section): + parser.add_section(section) + else: + parser = self.parser + section = repo + conffile = self.conffile + parser.set(section, option, value) + with open(conffile, "w") as f: + parser.write(f) + self.repos[repo][option] = value + + def sanity_check(self, initmode=False): + required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"] + if initmode: + required_options.remove("last_revision") + msg = "" + missing_options = [] + for name in self.repos: + for option in required_options: + if option not in self.repos[name]: + msg = "%s\nOption %s is not defined for component %s" %(msg, option, name) + missing_options.append(option) + # Sanitize dest_dir so that we do not have to deal with edge cases + # (unset, empty string, double slashes) in the rest of the code. + # It not being set will still be flagged as error because it is + # listed as required option above; that could be changed now. + dest_dir = os.path.normpath(self.repos[name].get("dest_dir", ".")) + self.repos[name]["dest_dir"] = "." if not dest_dir else dest_dir + if msg != "": + logger.error("configuration file %s has the following error: %s" % (self.conffile,msg)) + if self.localconffile and 'last_revision' in missing_options: + logger.error("local configuration file %s may be missing configuration for combo branch %s" % (self.localconffile, self.combobranch)) + sys.exit(1) + + # filterdiff is required by action_splitpatch, so check its availability + if subprocess.call("which filterdiff > /dev/null 2>&1", shell=True) != 0: + logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)") + sys.exit(1) + +def runcmd(cmd,destdir=None,printerr=True,out=None): + """ + execute command, raise CalledProcessError if fail + return output if succeed + """ + logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir)) + if not out: + out = os.tmpfile() + err = out + else: + err = os.tmpfile() + try: + subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir, shell=isinstance(cmd, str)) + except subprocess.CalledProcessError,e: + err.seek(0) + if printerr: + logger.error("%s" % err.read()) + raise e + + err.seek(0) + output = err.read() + logger.debug("output: %s" % output ) + return output + +def action_init(conf, args): + """ + Clone component repositories + Check git is initialised; if not, copy initial data from component repos + """ + for name in conf.repos: + ldir = conf.repos[name]['local_repo_dir'] + if not os.path.exists(ldir): + logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir)) + subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True) + if not os.path.exists(".git"): + runcmd("git init") + if conf.history: + # Need a common ref for all trees. + runcmd('git commit -m "initial empty commit" --allow-empty') + startrev = runcmd('git rev-parse master').strip() + + for name in conf.repos: + repo = conf.repos[name] + ldir = repo['local_repo_dir'] + branch = repo.get('branch', "master") + lastrev = repo.get('last_revision', None) + if lastrev and lastrev != "HEAD": + initialrev = lastrev + if branch: + if not check_rev_branch(name, ldir, lastrev, branch): + sys.exit(1) + logger.info("Copying data from %s at specified revision %s..." % (name, lastrev)) + else: + lastrev = None + initialrev = branch + logger.info("Copying data from %s..." % name) + # Sanity check initialrev and turn it into hash (required for copying history, + # because resolving a name ref only works in the component repo). + rev = runcmd('git rev-parse %s' % initialrev, ldir).strip() + if rev != initialrev: + try: + refs = runcmd('git show-ref -s %s' % initialrev, ldir).split('\n') + if len(set(refs)) > 1: + # Happens for example when configured to track + # "master" and there is a refs/heads/master. The + # traditional behavior from "git archive" (preserved + # here) it to choose the first one. This might not be + # intended, so at least warn about it. + logger.warn("%s: initial revision '%s' not unique, picking result of rev-parse = %s" % + (name, initialrev, refs[0])) + initialrev = rev + except: + # show-ref fails for hashes. Skip the sanity warning in that case. + pass + initialrev = rev + dest_dir = repo['dest_dir'] + if dest_dir != ".": + extract_dir = os.path.join(os.getcwd(), dest_dir) + if not os.path.exists(extract_dir): + os.makedirs(extract_dir) + else: + extract_dir = os.getcwd() + file_filter = repo.get('file_filter', "") + exclude_patterns = repo.get('file_exclude', '').split() + def copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir, + subdir=""): + # When working inside a filtered branch which had the + # files already moved, we need to prepend the + # subdirectory to all filters, otherwise they would + # not match. + if subdir == '.': + subdir = '' + elif subdir: + subdir = os.path.normpath(subdir) + file_filter = ' '.join([subdir + '/' + x for x in file_filter.split()]) + exclude_patterns = [subdir + '/' + x for x in exclude_patterns] + # To handle both cases, we cd into the target + # directory and optionally tell tar to strip the path + # prefix when the files were already moved. + subdir_components = len(subdir.split(os.path.sep)) if subdir else 0 + strip=('--strip-components=%d' % subdir_components) if subdir else '' + # TODO: file_filter wild cards do not work (and haven't worked before either), because + # a) GNU tar requires a --wildcards parameter before turning on wild card matching. + # b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c, + # in contrast to the other use of file_filter as parameter of "git archive" + # where it only matches .c files directly in src). + files = runcmd("git archive %s %s | tar -x -v %s -C %s %s" % + (initialrev, subdir, + strip, extract_dir, file_filter), + ldir) + if exclude_patterns: + # Implement file removal by letting tar create the + # file and then deleting it in the file system + # again. Uses the list of files created by tar (easier + # than walking the tree). + for file in files.split('\n'): + for pattern in exclude_patterns: + if fnmatch.fnmatch(file, pattern): + os.unlink(os.path.join(*([extract_dir] + ['..'] * subdir_components + [file]))) + break + + if not conf.history: + copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir) + else: + # First fetch remote history into local repository. + # We need a ref for that, so ensure that there is one. + refname = "combo-layer-init-%s" % name + runcmd("git branch -f %s %s" % (refname, initialrev), ldir) + runcmd("git fetch %s %s" % (ldir, refname)) + runcmd("git branch -D %s" % refname, ldir) + # Make that the head revision. + runcmd("git checkout -b %s %s" % (name, initialrev)) + # Optional: cut the history by replacing the given + # start point(s) with commits providing the same + # content (aka tree), but with commit information that + # makes it clear that this is an artifically created + # commit and nothing the original authors had anything + # to do with. + since_rev = repo.get('since_revision', '') + if since_rev: + committer = runcmd('git var GIT_AUTHOR_IDENT').strip() + # Same time stamp, no name. + author = re.sub('.* (\d+ [+-]\d+)', r'unknown <unknown> \1', committer) + logger.info('author %s' % author) + for rev in since_rev.split(): + # Resolve in component repo... + rev = runcmd('git log --oneline --no-abbrev-commit -n1 %s' % rev, ldir).split()[0] + # ... and then get the tree in current + # one. The commit should be in both repos with + # the same tree, but better check here. + tree = runcmd('git show -s --pretty=format:%%T %s' % rev).strip() + with tempfile.NamedTemporaryFile() as editor: + editor.write('''cat >$1 <<EOF +tree %s +author %s +committer %s + +%s: squashed import of component + +This commit copies the entire set of files as found in +%s %s + +For more information about previous commits, see the +upstream repository. + +Commit created by combo-layer. +EOF +''' % (tree, author, committer, name, name, since_rev)) + editor.flush() + os.environ['GIT_EDITOR'] = 'sh %s' % editor.name + runcmd('git replace --edit %s' % rev) + + # Optional: rewrite history to change commit messages or to move files. + if 'hook' in repo or dest_dir != ".": + filter_branch = ['git', 'filter-branch', '--force'] + with tempfile.NamedTemporaryFile() as hookwrapper: + if 'hook' in repo: + # Create a shell script wrapper around the original hook that + # can be used by git filter-branch. Hook may or may not have + # an absolute path. + hook = repo['hook'] + hook = os.path.join(os.path.dirname(conf.conffile), '..', hook) + # The wrappers turns the commit message + # from stdin into a fake patch header. + # This is good enough for changing Subject + # and commit msg body with normal + # combo-layer hooks. + hookwrapper.write('''set -e +tmpname=$(mktemp) +trap "rm $tmpname" EXIT +echo -n 'Subject: [PATCH] ' >>$tmpname +cat >>$tmpname +if ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then + echo >>$tmpname +fi +echo '---' >>$tmpname +%s $tmpname $GIT_COMMIT %s +tail -c +18 $tmpname | head -c -4 +''' % (hook, name)) + hookwrapper.flush() + filter_branch.extend(['--msg-filter', 'bash %s' % hookwrapper.name]) + if dest_dir != ".": + parent = os.path.dirname(dest_dir) + if not parent: + parent = '.' + # May run outside of the current directory, so do not assume that .git exists. + filter_branch.extend(['--tree-filter', 'mkdir -p .git/tmptree && find . -mindepth 1 -maxdepth 1 ! -name .git -print0 | xargs -0 -I SOURCE mv SOURCE .git/tmptree && mkdir -p %s && mv .git/tmptree %s' % (parent, dest_dir)]) + filter_branch.append('HEAD') + runcmd(filter_branch) + runcmd('git update-ref -d refs/original/refs/heads/%s' % name) + repo['rewritten_revision'] = runcmd('git rev-parse HEAD').strip() + repo['stripped_revision'] = repo['rewritten_revision'] + # Optional filter files: remove everything and re-populate using the normal filtering code. + # Override any potential .gitignore. + if file_filter or exclude_patterns: + runcmd('git rm -rf .') + if not os.path.exists(extract_dir): + os.makedirs(extract_dir) + copy_selected_files('HEAD', extract_dir, file_filter, exclude_patterns, '.', + subdir=dest_dir) + runcmd('git add --all --force .') + if runcmd('git status --porcelain'): + # Something to commit. + runcmd(['git', 'commit', '-m', + '''%s: select file subset + +Files from the component repository were chosen based on +the following filters: +file_filter = %s +file_exclude = %s''' % (name, file_filter or '<empty>', repo.get('file_exclude', '<empty>'))]) + repo['stripped_revision'] = runcmd('git rev-parse HEAD').strip() + + if not lastrev: + lastrev = runcmd('git rev-parse %s' % initialrev, ldir).strip() + conf.update(name, "last_revision", lastrev, initmode=True) + + if not conf.history: + runcmd("git add .") + else: + # Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies + runcmd('git checkout master') + merge = ['git', 'merge', '--no-commit'] + for name in conf.repos: + repo = conf.repos[name] + # Use branch created earlier. + merge.append(name) + # Root all commits which have no parent in the common + # ancestor in the new repository. + for start in runcmd('git log --pretty=format:%%H --max-parents=0 %s' % name).split('\n'): + runcmd('git replace --graft %s %s' % (start, startrev)) + try: + runcmd(merge) + except Exception, error: + logger.info('''Merging component repository history failed, perhaps because of merge conflicts. +It may be possible to commit anyway after resolving these conflicts. + +%s''' % error) + # Create MERGE_HEAD and MERGE_MSG. "git merge" itself + # does not create MERGE_HEAD in case of a (harmless) failure, + # and we want certain auto-generated information in the + # commit message for future reference and/or automation. + with open('.git/MERGE_HEAD', 'w') as head: + with open('.git/MERGE_MSG', 'w') as msg: + msg.write('repo: initial import of components\n\n') + # head.write('%s\n' % startrev) + for name in conf.repos: + repo = conf.repos[name] + # <upstream ref> <rewritten ref> <rewritten + files removed> + msg.write('combo-layer-%s: %s %s %s\n' % (name, + repo['last_revision'], + repo['rewritten_revision'], + repo['stripped_revision'])) + rev = runcmd('git rev-parse %s' % name).strip() + head.write('%s\n' % rev) + + if conf.localconffile: + localadded = True + try: + runcmd("git rm --cached %s" % conf.localconffile, printerr=False) + except subprocess.CalledProcessError: + localadded = False + if localadded: + localrelpath = os.path.relpath(conf.localconffile) + runcmd("grep -q %s .gitignore || echo %s >> .gitignore" % (localrelpath, localrelpath)) + runcmd("git add .gitignore") + logger.info("Added local configuration file %s to .gitignore", localrelpath) + logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.") + else: + logger.info("Repository already initialised, nothing to do.") + + +def check_repo_clean(repodir): + """ + check if the repo is clean + exit if repo is dirty + """ + output=runcmd("git status --porcelain", repodir) + r = re.compile('\?\? patch-.*/') + dirtyout = [item for item in output.splitlines() if not r.match(item)] + if dirtyout: + logger.error("git repo %s is dirty, please fix it first", repodir) + sys.exit(1) + +def check_patch(patchfile): + f = open(patchfile) + ln = f.readline() + of = None + in_patch = False + beyond_msg = False + pre_buf = '' + while ln: + if not beyond_msg: + if ln == '---\n': + if not of: + break + in_patch = False + beyond_msg = True + elif ln.startswith('--- '): + # We have a diff in the commit message + in_patch = True + if not of: + print('WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile) + of = open(patchfile + '.tmp', 'w') + of.write(pre_buf) + pre_buf = '' + elif in_patch and not ln[0] in '+-@ \n\r': + in_patch = False + if of: + if in_patch: + of.write(' ' + ln) + else: + of.write(ln) + else: + pre_buf += ln + ln = f.readline() + f.close() + if of: + of.close() + os.rename(patchfile + '.tmp', patchfile) + +def drop_to_shell(workdir=None): + if not sys.stdin.isatty(): + print "Not a TTY so can't drop to shell for resolution, exiting." + return False + + shell = os.environ.get('SHELL', 'bash') + print('Dropping to shell "%s"\n' \ + 'When you are finished, run the following to continue:\n' \ + ' exit -- continue to apply the patches\n' \ + ' exit 1 -- abort\n' % shell); + ret = subprocess.call([shell], cwd=workdir) + if ret != 0: + print "Aborting" + return False + else: + return True + +def check_rev_branch(component, repodir, rev, branch): + try: + actualbranch = runcmd("git branch --contains %s" % rev, repodir, printerr=False) + except subprocess.CalledProcessError as e: + if e.returncode == 129: + actualbranch = "" + else: + raise + + if not actualbranch: + logger.error("%s: specified revision %s is invalid!" % (component, rev)) + return False + + branches = [] + branchlist = actualbranch.split("\n") + for b in branchlist: + branches.append(b.strip().split(' ')[-1]) + + if branch not in branches: + logger.error("%s: specified revision %s is not on specified branch %s!" % (component, rev, branch)) + return False + return True + +def get_repos(conf, repo_names): + repos = [] + for name in repo_names: + if name.startswith('-'): + break + else: + repos.append(name) + for repo in repos: + if not repo in conf.repos: + logger.error("Specified component '%s' not found in configuration" % repo) + sys.exit(1) + + if not repos: + repos = [ repo for repo in conf.repos if conf.repos[repo].get("update", True) ] + + return repos + +def action_pull(conf, args): + """ + update the component repos only + """ + repos = get_repos(conf, args[1:]) + + # make sure all repos are clean + for name in repos: + check_repo_clean(conf.repos[name]['local_repo_dir']) + + for name in repos: + repo = conf.repos[name] + ldir = repo['local_repo_dir'] + branch = repo.get('branch', "master") + logger.info("update branch %s of component repo %s in %s ..." % (branch, name, ldir)) + if not conf.hard_reset: + # Try to pull only the configured branch. Beware that this may fail + # when the branch is currently unknown (for example, after reconfiguring + # combo-layer). In that case we need to fetch everything and try the check out + # and pull again. + try: + runcmd("git checkout %s" % branch, ldir, printerr=False) + except subprocess.CalledProcessError: + output=runcmd("git fetch", ldir) + logger.info(output) + runcmd("git checkout %s" % branch, ldir) + runcmd("git pull --ff-only", ldir) + else: + output=runcmd("git pull --ff-only", ldir) + logger.info(output) + else: + output=runcmd("git fetch", ldir) + logger.info(output) + runcmd("git checkout %s" % branch, ldir) + runcmd("git reset --hard FETCH_HEAD", ldir) + +def action_update(conf, args): + """ + update the component repos + generate the patch list + apply the generated patches + """ + components = [arg.split(':')[0] for arg in args[1:]] + revisions = {} + for arg in args[1:]: + if ':' in arg: + a = arg.split(':', 1) + revisions[a[0]] = a[1] + repos = get_repos(conf, components) + + # make sure combo repo is clean + check_repo_clean(os.getcwd()) + + import uuid + patch_dir = "patch-%s" % uuid.uuid4() + if not os.path.exists(patch_dir): + os.mkdir(patch_dir) + + # Step 1: update the component repos + if conf.nopull: + logger.info("Skipping pull (-n)") + else: + action_pull(conf, ['arg0'] + components) + + for name in repos: + revision = revisions.get(name, None) + repo = conf.repos[name] + ldir = repo['local_repo_dir'] + dest_dir = repo['dest_dir'] + branch = repo.get('branch', "master") + repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name) + + # Step 2: generate the patch list and store to patch dir + logger.info("Generating patches from %s..." % name) + top_revision = revision or branch + if not check_rev_branch(name, ldir, top_revision, branch): + sys.exit(1) + if dest_dir != ".": + prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir) + else: + prefix = "" + if repo['last_revision'] == "": + logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name) + patch_cmd_range = "--root %s" % top_revision + rev_cmd_range = top_revision + else: + if not check_rev_branch(name, ldir, repo['last_revision'], branch): + sys.exit(1) + patch_cmd_range = "%s..%s" % (repo['last_revision'], top_revision) + rev_cmd_range = patch_cmd_range + + file_filter = repo.get('file_filter',".") + + # Filter out unwanted files + exclude = repo.get('file_exclude', '') + if exclude: + for path in exclude.split(): + p = "%s/%s" % (dest_dir, path) if dest_dir != '.' else path + file_filter += " ':!%s'" % p + + patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \ + (prefix,repo_patch_dir, patch_cmd_range, file_filter) + output = runcmd(patch_cmd, ldir) + logger.debug("generated patch set:\n%s" % output) + patchlist = output.splitlines() + + rev_cmd = "git rev-list --no-merges %s -- %s" % (rev_cmd_range, file_filter) + revlist = runcmd(rev_cmd, ldir).splitlines() + + # Step 3: Call repo specific hook to adjust patch + if 'hook' in repo: + # hook parameter is: ./hook patchpath revision reponame + count=len(revlist)-1 + for patch in patchlist: + runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name)) + count=count-1 + + # Step 4: write patch list and revision list to file, for user to edit later + patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name) + repo['patchlist'] = patchlist_file + f = open(patchlist_file, 'w') + count=len(revlist)-1 + for patch in patchlist: + f.write("%s %s\n" % (patch, revlist[count])) + check_patch(os.path.join(patch_dir, patch)) + count=count-1 + f.close() + + # Step 5: invoke bash for user to edit patch and patch list + if conf.interactive: + print('You may now edit the patch and patch list in %s\n' \ + 'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir); + if not drop_to_shell(patch_dir): + sys.exit(1) + + # Step 6: apply the generated and revised patch + apply_patchlist(conf, repos) + runcmd("rm -rf %s" % patch_dir) + + # Step 7: commit the updated config file if it's being tracked + relpath = os.path.relpath(conf.conffile) + try: + output = runcmd("git status --porcelain %s" % relpath, printerr=False) + except: + # Outside the repository + output = None + if output: + logger.info("Committing updated configuration file") + if output.lstrip().startswith("M"): + + # create the "components" string + component_str = "all components" + if len(components) > 0: + # otherwise tell which components were actually changed + component_str = ", ".join(components) + + # expand the template with known values + template = Template(conf.commit_msg_template) + raw_msg = template.substitute(components = component_str) + + # sanitize the string before using it in command line + msg = raw_msg.replace('"', '\\"') + + runcmd('git commit -m "%s" %s' % (msg, relpath)) + +def apply_patchlist(conf, repos): + """ + apply the generated patch list to combo repo + """ + for name in repos: + repo = conf.repos[name] + lastrev = repo["last_revision"] + prevrev = lastrev + + # Get non-blank lines from patch list file + patchlist = [] + if os.path.exists(repo['patchlist']) or not conf.interactive: + # Note: we want this to fail here if the file doesn't exist and we're not in + # interactive mode since the file should exist in this case + with open(repo['patchlist']) as f: + for line in f: + line = line.rstrip() + if line: + patchlist.append(line) + + ldir = conf.repos[name]['local_repo_dir'] + branch = conf.repos[name].get('branch', "master") + branchrev = runcmd("git rev-parse %s" % branch, ldir).strip() + + if patchlist: + logger.info("Applying patches from %s..." % name) + linecount = len(patchlist) + i = 1 + for line in patchlist: + patchfile = line.split()[0] + lastrev = line.split()[1] + patchdisp = os.path.relpath(patchfile) + if os.path.getsize(patchfile) == 0: + logger.info("(skipping %d/%d %s - no changes)" % (i, linecount, patchdisp)) + else: + cmd = "git am --keep-cr %s-p1 %s" % ('-s ' if repo.get('signoff', True) else '', patchfile) + logger.info("Applying %d/%d: %s" % (i, linecount, patchdisp)) + try: + runcmd(cmd) + except subprocess.CalledProcessError: + logger.info('Running "git am --abort" to cleanup repo') + runcmd("git am --abort") + logger.error('"%s" failed' % cmd) + logger.info("Please manually apply patch %s" % patchdisp) + logger.info("Note: if you exit and continue applying without manually applying the patch, it will be skipped") + if not drop_to_shell(): + if prevrev != repo['last_revision']: + conf.update(name, "last_revision", prevrev) + sys.exit(1) + prevrev = lastrev + i += 1 + # Once all patches are applied, we should update + # last_revision to the branch head instead of the last + # applied patch. The two are not necessarily the same when + # the last commit is a merge commit or when the patches at + # the branch head were intentionally excluded. + # + # If we do not do that for a merge commit, the next + # combo-layer run will only exclude patches reachable from + # one of the merged branches and try to re-apply patches + # from other branches even though they were already + # copied. + # + # If patches were intentionally excluded, the next run will + # present them again instead of skipping over them. This + # may or may not be intended, so the code here is conservative + # and only addresses the "head is merge commit" case. + if lastrev != branchrev and \ + len(runcmd("git show --pretty=format:%%P --no-patch %s" % branch, ldir).split()) > 1: + lastrev = branchrev + else: + logger.info("No patches to apply from %s" % name) + lastrev = branchrev + + if lastrev != repo['last_revision']: + conf.update(name, "last_revision", lastrev) + +def action_splitpatch(conf, args): + """ + generate the commit patch and + split the patch per repo + """ + logger.debug("action_splitpatch") + if len(args) > 1: + commit = args[1] + else: + commit = "HEAD" + patchdir = "splitpatch-%s" % commit + if not os.path.exists(patchdir): + os.mkdir(patchdir) + + # filerange_root is for the repo whose dest_dir is root "." + # and it should be specified by excluding all other repo dest dir + # like "-x repo1 -x repo2 -x repo3 ..." + filerange_root = "" + for name in conf.repos: + dest_dir = conf.repos[name]['dest_dir'] + if dest_dir != ".": + filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir) + + for name in conf.repos: + dest_dir = conf.repos[name]['dest_dir'] + patch_filename = "%s/%s.patch" % (patchdir, name) + if dest_dir == ".": + cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename) + else: + cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename) + runcmd(cmd) + # Detect empty patches (including those produced by filterdiff above + # that contain only preamble text) + if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "": + os.remove(patch_filename) + logger.info("(skipping %s - no changes)", name) + else: + logger.info(patch_filename) + +def action_error(conf, args): + logger.info("invalid action %s" % args[0]) + +actions = { + "init": action_init, + "update": action_update, + "pull": action_pull, + "splitpatch": action_splitpatch, +} + +def main(): + parser = optparse.OptionParser( + version = "Combo Layer Repo Tool version %s" % __version__, + usage = """%prog [options] action + +Create and update a combination layer repository from multiple component repositories. + +Action: + init initialise the combo layer repo + update [components] get patches from component repos and apply them to the combo repo + pull [components] just pull component repos only + splitpatch [commit] generate commit patch and split per component, default commit is HEAD""") + + parser.add_option("-c", "--conf", help = "specify the config file (conf/combo-layer.conf is the default).", + action = "store", dest = "conffile", default = "conf/combo-layer.conf") + + parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches", + action = "store_true", dest = "interactive", default = False) + + parser.add_option("-D", "--debug", help = "output debug information", + action = "store_true", dest = "debug", default = False) + + parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update", + action = "store_true", dest = "nopull", default = False) + + parser.add_option("--hard-reset", + help = "instead of pull do fetch and hard-reset in component repos", + action = "store_true", dest = "hard_reset", default = False) + + parser.add_option("-H", "--history", help = "import full history of components during init", + action = "store_true", default = False) + + options, args = parser.parse_args(sys.argv) + + # Dispatch to action handler + if len(args) == 1: + logger.error("No action specified, exiting") + parser.print_help() + elif args[1] not in actions: + logger.error("Unsupported action %s, exiting\n" % (args[1])) + parser.print_help() + elif not os.path.exists(options.conffile): + logger.error("No valid config file, exiting\n") + parser.print_help() + else: + if options.debug: + logger.setLevel(logging.DEBUG) + confdata = Configuration(options) + initmode = (args[1] == 'init') + confdata.sanity_check(initmode) + actions.get(args[1], action_error)(confdata, args[1:]) + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/scripts/combo-layer-hook-default.sh b/scripts/combo-layer-hook-default.sh new file mode 100755 index 0000000..1e3a3b9 --- /dev/null +++ b/scripts/combo-layer-hook-default.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Hook to add source component/revision info to commit message +# Parameter: +# $1 patch-file +# $2 revision +# $3 reponame + +patchfile=$1 +rev=$2 +reponame=$3 + +sed -i -e "0,/^Subject:/s#^Subject: \[PATCH\] \($reponame: \)*\(.*\)#Subject: \[PATCH\] $reponame: \2#" $patchfile +if grep -q '^Signed-off-by:' $patchfile; then + # Insert before Signed-off-by. + sed -i -e "0,/^Signed-off-by:/s#\(^Signed-off-by:.*\)#\(From $reponame rev: $rev\)\n\n\1#" $patchfile +else + # Insert before final --- separator, with extra blank lines removed. + perl -e "\$_ = join('', <>); s/^(.*\S[ \t]*)(\n|\n\s*\n)---\n/\$1\n\nFrom $reponame rev: $rev\n---\n/s; print;" $patchfile >$patchfile.tmp + mv $patchfile.tmp $patchfile +fi diff --git a/scripts/combo-layer.conf.example b/scripts/combo-layer.conf.example new file mode 100644 index 0000000..90e2b58 --- /dev/null +++ b/scripts/combo-layer.conf.example @@ -0,0 +1,93 @@ +# combo-layer example configuration file + +# Default values for all sections. +[DEFAULT] + +# Add 'Signed-off-by' to all commits that get imported automatically. +signoff = True + +# component name +[bitbake] + +# Override signedoff default above (not very useful, but possible). +signoff = False + +# mandatory options +# git upstream uri +src_uri = git://git.openembedded.org/bitbake + +# the directory to clone the component repo +local_repo_dir = /home/kyu3/src/test/bitbake + +# the relative dir within the combo repo to put the component files +# use "." if the files should be in the root dir +dest_dir = bitbake + +# the last update revision. +# "init" will set this to the latest revision automatically, however if it +# is empty when "update" is run, the tool will start from the first commit. +# Note that this value will get updated by "update" if the component repo's +# latest revision changed and the operation completes successfully. +last_revision = + +# optional options: + +# branch: specify the branch in the component repo to pull from +# (master if not specified) + +# file_filter: only include the specified file(s) +# file_filter = [path] [path] ... +# example: +# file_filter = src/ : only include the subdir src +# file_filter = src/*.c : only include the src *.c file +# file_filter = src/main.c src/Makefile.am : only include these two files + +# file_exclude: filter out these file(s) +# file_exclude = [path] [path] ... +# +# Each entry must match a file name. In contrast do file_filter, matching +# a directory has no effect. To achieve that, use append a * wildcard +# at the end. +# +# Wildcards are applied to the complete path and also match slashes. +# +# example: +# file_exclude = src/foobar/* : exclude everything under src/foobar +# file_exclude = src/main.c : filter out main.c after including it with file_filter = src/*.c +# file_exclude = *~ : exclude backup files + +# hook: if provided, the tool will call the hook to process the generated +# patch from upstream, and then apply the modified patch to the combo +# repo. +# the hook script is called as follows: ./hook patchpath revision reponame +# example: +# hook = combo-layer-hook-default.sh + +# since_revision: +# since_revision = release-1-2 +# since_revision = 12345 abcdf +# +# If provided, truncate imported history during "combo-layer --history +# init" at the specified revision(s). More than one can be specified +# to cut off multiple component branches. +# +# The specified commits themselves do not get imported. Instead, an +# artificial commit with "unknown" author is created with a content +# that matches the original commit. + +[oe-core] +src_uri = git://git.openembedded.org/openembedded-core +local_repo_dir = /home/kyu3/src/test/oecore +dest_dir = . +last_revision = +since_revision = some-tag-or-commit-on-master-branch + +# It is also possible to embed python code in the config values. Similar +# to bitbake it considers every value starting with @ to be a python +# script. +# e.g. local_repo_dir could easily be configured using an environment +# variable: +# +# [bitbake] +# local_repo_dir = @os.getenv("LOCAL_REPO_DIR") + "/bitbake" +# diff --git a/scripts/contrib/bb-perf/bb-matrix-plot.sh b/scripts/contrib/bb-perf/bb-matrix-plot.sh new file mode 100755 index 0000000..136a255 --- /dev/null +++ b/scripts/contrib/bb-perf/bb-matrix-plot.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Copyright (c) 2011, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# DESCRIPTION +# This script operates on the .dat file generated by bb-matrix.sh. It tolerates +# the header by skipping the first line, but error messages and bad data records +# need to be removed first. It will generate three views of the plot, and leave +# an interactive view open for further analysis. +# +# AUTHORS +# Darren Hart <dvhart@linux.intel.com> +# + +# Setup the defaults +DATFILE="bb-matrix.dat" +XLABEL="BB_NUMBER_THREADS" +YLABEL="PARALLEL_MAKE" +FIELD=3 +DEF_TITLE="Elapsed Time (seconds)" +PM3D_FRAGMENT="unset surface; set pm3d at s hidden3d 100" +SIZE="640,480" + +function usage { +CMD=$(basename $0) +cat <<EOM +Usage: $CMD [-d datfile] [-f field] [-h] [-t title] [-w] + -d datfile The data file generated by bb-matrix.sh (default: $DATFILE) + -f field The field index to plot as the Z axis from the data file + (default: $FIELD, "$DEF_TITLE") + -h Display this help message + -s W,H PNG and window size in pixels (default: $SIZE) + -t title The title to display, should describe the field (-f) and units + (default: "$DEF_TITLE") + -w Render the plot as wireframe with a 2D colormap projected on the + XY plane rather than as the texture for the surface +EOM +} + +# Parse and validate arguments +while getopts "d:f:hs:t:w" OPT; do + case $OPT in + d) + DATFILE="$OPTARG" + ;; + f) + FIELD="$OPTARG" + ;; + h) + usage + exit 0 + ;; + s) + SIZE="$OPTARG" + ;; + t) + TITLE="$OPTARG" + ;; + w) + PM3D_FRAGMENT="set pm3d at b" + W="-w" + ;; + *) + usage + exit 1 + ;; + esac +done + +# Ensure the data file exists +if [ ! -f "$DATFILE" ]; then + echo "ERROR: $DATFILE does not exist" + usage + exit 1 +fi +PLOT_BASENAME=${DATFILE%.*}-f$FIELD$W + +# Set a sane title +# TODO: parse the header and define titles for each format parameter for TIME(1) +if [ -z "$TITLE" ]; then + if [ ! "$FIELD" == "3" ]; then + TITLE="Field $FIELD" + else + TITLE="$DEF_TITLE" + fi +fi + +# Determine the dgrid3d mesh dimensions size +MIN=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 1 | sed 's/^0*//' | sort -n | uniq | head -n1) +MAX=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 1 | sed 's/^0*//' | sort -n | uniq | tail -n1) +BB_CNT=$[${MAX} - $MIN + 1] +MIN=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 2 | sed 's/^0*//' | sort -n | uniq | head -n1) +MAX=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 2 | sed 's/^0*//' | sort -n | uniq | tail -n1) +PM_CNT=$[${MAX} - $MIN + 1] + + +(cat <<EOF +set title "$TITLE" +set xlabel "$XLABEL" +set ylabel "$YLABEL" +set style line 100 lt 5 lw 1.5 +$PM3D_FRAGMENT +set dgrid3d $PM_CNT,$BB_CNT splines +set ticslevel 0.2 + +set term png size $SIZE +set output "$PLOT_BASENAME.png" +splot "$DATFILE" every ::1 using 1:2:$FIELD with lines ls 100 + +set view 90,0 +set output "$PLOT_BASENAME-bb.png" +replot + +set view 90,90 +set output "$PLOT_BASENAME-pm.png" +replot + +set view 60,30 +set term wxt size $SIZE +replot +EOF +) | gnuplot --persist diff --git a/scripts/contrib/bb-perf/bb-matrix.sh b/scripts/contrib/bb-perf/bb-matrix.sh new file mode 100755 index 0000000..1064565 --- /dev/null +++ b/scripts/contrib/bb-perf/bb-matrix.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Copyright (c) 2011, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# DESCRIPTION +# This script runs BB_CMD (typically building core-image-sato) for all +# combincations of BB_RANGE and PM_RANGE values. It saves off all the console +# logs, the buildstats directories, and creates a bb-pm-runtime.dat file which +# can be used to postprocess the results with a plotting tool, spreadsheet, etc. +# Before running this script, it is recommended that you pre-download all the +# necessary sources by performing the BB_CMD once manually. It is also a good +# idea to disable cron to avoid runtime variations caused by things like the +# locate process. Be sure to sanitize the dat file prior to post-processing as +# it may contain error messages or bad runs that should be removed. +# +# AUTHORS +# Darren Hart <dvhart@linux.intel.com> +# + +# The following ranges are appropriate for a 4 core system with 8 logical units +# Use leading 0s to ensure all digits are the same string length, this results +# in nice log file names and columnar dat files. +BB_RANGE="04 05 06 07 08 09 10 11 12 13 14 15 16" +PM_RANGE="04 05 06 07 08 09 10 11 12 13 14 15 16" + +DATADIR="bb-matrix-$$" +BB_CMD="bitbake core-image-minimal" +RUNTIME_LOG="$DATADIR/bb-matrix.dat" + +# See TIME(1) for a description of the time format parameters +# The following all report 0: W K r s t w +TIME_STR="%e %S %U %P %c %w %R %F %M %x" + +# Prepare the DATADIR +mkdir $DATADIR +if [ $? -ne 0 ]; then + echo "Failed to create $DATADIR." + exit 1 +fi + +# Add a simple header +echo "BB PM $TIME_STR" > $RUNTIME_LOG +for BB in $BB_RANGE; do + for PM in $PM_RANGE; do + RUNDIR="$DATADIR/$BB-$PM-build" + mkdir $RUNDIR + BB_LOG=$RUNDIR/$BB-$PM-bitbake.log + date + echo "BB=$BB PM=$PM Logging to $BB_LOG" + + echo -n " Preparing the work directory... " + rm -rf pseudodone tmp sstate-cache tmp-eglibc &> /dev/null + echo "done" + + # Export the variables under test and run the bitbake command + # Strip any leading zeroes before passing to bitbake + export BB_NUMBER_THREADS=$(echo $BB | sed 's/^0*//') + export PARALLEL_MAKE="-j $(echo $PM | sed 's/^0*//')" + /usr/bin/time -f "$BB $PM $TIME_STR" -a -o $RUNTIME_LOG $BB_CMD &> $BB_LOG + + echo " $(tail -n1 $RUNTIME_LOG)" + cp -a tmp/buildstats $RUNDIR/$BB-$PM-buildstats + done +done diff --git a/scripts/contrib/bb-perf/buildstats.sh b/scripts/contrib/bb-perf/buildstats.sh new file mode 100755 index 0000000..96158a9 --- /dev/null +++ b/scripts/contrib/bb-perf/buildstats.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# Copyright (c) 2011, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# DESCRIPTION +# Given a 'buildstats' path (created by bitbake when setting +# USER_CLASSES ?= "buildstats" on local.conf) and task names, outputs +# '<task> <recipe> <elapsed time>' for all recipes. Elapsed times are in +# seconds, and task should be given without the 'do_' prefix. +# +# Some useful pipelines +# +# 1. Tasks with largest elapsed times +# $ buildstats.sh -b <buildstats> | sort -k3 -n -r | head +# +# 2. Min, max, sum per task (in needs GNU datamash) +# $ buildstats.sh -b <buildstats> | datamash -t' ' -g1 min 3 max 3 sum 3 | sort -k4 -n -r +# +# AUTHORS +# Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> +# +BS_DIR="tmp/buildstats" +TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" + +function usage { +CMD=$(basename $0) +cat <<EOM +Usage: $CMD [-b buildstats_dir] [-t do_task] + -b buildstats The path where the folder resides + (default: "$BS_DIR") + -t tasks The tasks to be computed + (default: "$TASKS") + -h Display this help message +EOM +} + +# Parse and validate arguments +while getopts "b:t:h" OPT; do + case $OPT in + b) + BS_DIR="$OPTARG" + ;; + t) + TASKS="$OPTARG" + ;; + h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + +# Ensure the buildstats folder exists +if [ ! -d "$BS_DIR" ]; then + echo "ERROR: $BS_DIR does not exist" + usage + exit 1 +fi + +RECIPE_FIELD=1 +TIME_FIELD=4 + +tasks=(${TASKS//:/ }) +for task in "${tasks[@]}"; do + task="do_${task}" + for file in $(find ${BS_DIR} -type f -name ${task}); do + recipe=$(sed -n -e "/$task/p" ${file} | cut -d ':' -f${RECIPE_FIELD}) + time=$(sed -n -e "/$task/p" ${file} | cut -d ':' -f${TIME_FIELD} | cut -d ' ' -f2) + echo "${task} ${recipe} ${time}" + done +done diff --git a/scripts/contrib/bbvars.py b/scripts/contrib/bbvars.py new file mode 100755 index 0000000..0896d64 --- /dev/null +++ b/scripts/contrib/bbvars.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# Copyright (C) Darren Hart <dvhart@linux.intel.com>, 2010 + + +import sys +import getopt +import os +import os.path +import re + +def usage(): + print 'Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0]) + print ' -d FILENAME documentation file to search' + print ' -h, --help display this help and exit' + print ' -m METADIR meta directory to search for recipes' + print ' -t FILENAME documentation config file (for doc tags)' + print ' -T Only display variables with doc tags (requires -t)' + +def recipe_bbvars(recipe): + ''' Return a unique set of every bbvar encountered in the recipe ''' + prog = re.compile("[A-Z_]+") + vset = set() + try: + r = open(recipe) + except IOError as (errno, strerror): + print 'WARNING: Failed to open recipe ', recipe + print strerror + + for line in r: + # Strip any comments from the line + line = line.rsplit('#')[0] + vset = vset.union(set(prog.findall(line))) + r.close() + + bbvars = {} + for v in vset: + bbvars[v] = 1 + + return bbvars + +def collect_bbvars(metadir): + ''' Walk the metadir and collect the bbvars from each recipe found ''' + bbvars = {} + for root,dirs,files in os.walk(metadir): + for name in files: + if name.find(".bb") >= 0: + for key in recipe_bbvars(os.path.join(root,name)).iterkeys(): + if bbvars.has_key(key): + bbvars[key] = bbvars[key] + 1 + else: + bbvars[key] = 1 + return bbvars + +def bbvar_is_documented(var, docfiles): + prog = re.compile(".*($|[^A-Z_])%s([^A-Z_]|$)" % (var)) + for doc in docfiles: + try: + f = open(doc) + except IOError as (errno, strerror): + print 'WARNING: Failed to open doc ', doc + print strerror + for line in f: + if prog.match(line): + return True + f.close() + return False + +def bbvar_doctag(var, docconf): + prog = re.compile('^%s\[doc\] *= *"(.*)"' % (var)) + if docconf == "": + return "?" + + try: + f = open(docconf) + except IOError as (errno, strerror): + return strerror + + for line in f: + m = prog.search(line) + if m: + return m.group(1) + + f.close() + return "" + +def main(): + docfiles = [] + metadirs = [] + bbvars = {} + undocumented = [] + docconf = "" + onlydoctags = False + + # Collect and validate input + try: + opts, args = getopt.getopt(sys.argv[1:], "d:hm:t:T", ["help"]) + except getopt.GetoptError, err: + print '%s' % str(err) + usage() + sys.exit(2) + + for o, a in opts: + if o in ('-h', '--help'): + usage() + sys.exit(0) + elif o == '-d': + if os.path.isfile(a): + docfiles.append(a) + else: + print 'ERROR: documentation file %s is not a regular file' % (a) + sys.exit(3) + elif o == '-m': + if os.path.isdir(a): + metadirs.append(a) + else: + print 'ERROR: meta directory %s is not a directory' % (a) + sys.exit(4) + elif o == "-t": + if os.path.isfile(a): + docconf = a + elif o == "-T": + onlydoctags = True + else: + assert False, "unhandled option" + + if len(docfiles) == 0: + print 'ERROR: no docfile specified' + usage() + sys.exit(5) + + if len(metadirs) == 0: + print 'ERROR: no metadir specified' + usage() + sys.exit(6) + + if onlydoctags and docconf == "": + print 'ERROR: no docconf specified' + usage() + sys.exit(7) + + # Collect all the variable names from the recipes in the metadirs + for m in metadirs: + for key,cnt in collect_bbvars(m).iteritems(): + if bbvars.has_key(key): + bbvars[key] = bbvars[key] + cnt + else: + bbvars[key] = cnt + + # Check each var for documentation + varlen = 0 + for v in bbvars.iterkeys(): + if len(v) > varlen: + varlen = len(v) + if not bbvar_is_documented(v, docfiles): + undocumented.append(v) + undocumented.sort() + varlen = varlen + 1 + + # Report all undocumented variables + print 'Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars)) + header = '%s%s%s' % (str("VARIABLE").ljust(varlen), str("COUNT").ljust(6), str("DOCTAG").ljust(7)) + print header + print str("").ljust(len(header), '=') + for v in undocumented: + doctag = bbvar_doctag(v, docconf) + if not onlydoctags or not doctag == "": + print '%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag) + + +if __name__ == "__main__": + main() diff --git a/scripts/contrib/build-perf-test.sh b/scripts/contrib/build-perf-test.sh new file mode 100755 index 0000000..7d99228 --- /dev/null +++ b/scripts/contrib/build-perf-test.sh @@ -0,0 +1,400 @@ +#!/bin/bash +# +# This script runs a series of tests (with and without sstate) and reports build time (and tmp/ size) +# +# Build performance test script +# +# Copyright 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# +# AUTHORS: +# Stefan Stanacar <stefanx.stanacar@intel.com> + + +ME=$(basename $0) + +# +# usage and setup +# + +usage () { +cat << EOT +Usage: $ME [-h] + $ME [-c <commit>] [-v] [-m <val>] [-j <val>] [-t <val>] [-i <image-name>] [-d <path>] +Options: + -h + Display this help and exit. + -c <commit> + git checkout <commit> before anything else + -v + Show bitbake output, don't redirect it to a log. + -m <machine> + Value for MACHINE. Default is qemux86. + -j <val> + Value for PARALLEL_MAKE. Default is 8. + -t <val> + Value for BB_NUMBER_THREADS. Default is 8. + -i <image-name> + Instead of timing against core-image-sato, use <image-name> + -d <path> + Use <path> as DL_DIR + -p <githash> + Cherry pick githash onto the commit + +Note: current working directory must be inside a poky git clone. + +EOT +} + + +if clonedir=$(git rev-parse --show-toplevel); then + cd $clonedir +else + echo "The current working dir doesn't seem to be a poky git clone. Please cd there before running $ME" + exit 1 +fi + +IMAGE="core-image-sato" +verbose=0 +dldir= +commit= +pmake= +cherrypicks= +while getopts "hvc:m:j:t:i:d:p:" opt; do + case $opt in + h) usage + exit 0 + ;; + v) verbose=1 + ;; + c) commit=$OPTARG + ;; + m) export MACHINE=$OPTARG + ;; + j) pmake=$OPTARG + ;; + t) export BB_NUMBER_THREADS=$OPTARG + ;; + i) IMAGE=$OPTARG + ;; + d) dldir=$OPTARG + ;; + p) cherrypicks="$cherrypicks $OPTARG" + ;; + *) usage + exit 1 + ;; + esac +done + + +#drop cached credentials and test for sudo access without a password +sudo -k -n ls > /dev/null 2>&1 +reqpass=$? +if [ $reqpass -ne 0 ]; then + echo "The script requires sudo access to drop caches between builds (echo 3 > /proc/sys/vm/drop_caches)" + read -s -p "Please enter your sudo password: " pass + echo +fi + +if [ -n "$commit" ]; then + echo "git checkout -f $commit" + git pull > /dev/null 2>&1 + git checkout -f $commit || exit 1 + git pull > /dev/null 2>&1 +fi + +if [ -n "$cherrypicks" ]; then + for c in $cherrypicks; do + git cherry-pick $c + done +fi + +rev=$(git rev-parse --short HEAD) || exit 1 +OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`" +BUILDDIR="$OUTDIR/build" +resultsfile="$OUTDIR/results.log" +cmdoutput="$OUTDIR/commands.log" +myoutput="$OUTDIR/output.log" +globalres="$clonedir/build-perf-test/globalres.log" + +mkdir -p $OUTDIR || exit 1 + +log () { + local msg="$1" + echo "`date`: $msg" | tee -a $myoutput +} + + +# +# Config stuff +# + +branch=`git branch 2>&1 | grep "^* " | tr -d "* "` +gitcommit=$(git rev-parse HEAD) || exit 1 +log "Running on $branch:$gitcommit" + +source ./oe-init-build-env $OUTDIR/build >/dev/null || exit 1 +cd $OUTDIR/build + +[ -n "$MACHINE" ] || export MACHINE="qemux86" +[ -n "$BB_NUMBER_THREADS" ] || export BB_NUMBER_THREADS="8" + +if [ -n "$pmake" ]; then + export PARALLEL_MAKE="-j $pmake" +else + export PARALLEL_MAKE="-j 8" +fi + +if [ -n "$dldir" ]; then + echo "DL_DIR = \"$dldir\"" >> conf/local.conf +else + echo "DL_DIR = \"$clonedir/build-perf-test/downloads\"" >> conf/local.conf +fi + +# Sometimes I've noticed big differences in timings for the same commit, on the same machine +# Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy) +echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf + + +# +# Functions +# + +declare -a TIMES +time_count=0 +declare -a SIZES +size_count=0 + +time_cmd () { + log " Timing: $*" + + if [ $verbose -eq 0 ]; then + /usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput + else + /usr/bin/time -v -o $resultsfile "$@" + fi + ret=$? + if [ $ret -eq 0 ]; then + t=`grep wall $resultsfile | sed 's/.*m:ss): //'` + log " TIME: $t" + TIMES[(( time_count++ ))]="$t" + else + log "ERROR: exit status was non-zero, will report time as 0." + TIMES[(( time_count++ ))]="0" + fi + + #time by default overwrites the output file and we want to keep the results + #it has an append option but I don't want to clobber the results in the same file + i=`ls $OUTDIR/results.log* |wc -l` + mv $resultsfile "${resultsfile}.${i}" + log "More stats can be found in ${resultsfile}.${i}" +} + +bbtime () { + time_cmd bitbake "$@" +} + +#we don't time bitbake here +bbnotime () { + local arg="$@" + log " Running: bitbake ${arg}" + if [ $verbose -eq 0 ]; then + bitbake ${arg} >> $cmdoutput + else + bitbake ${arg} + fi + ret=$? + if [ $ret -eq 0 ]; then + log " Finished bitbake ${arg}" + else + log "ERROR: exit status was non-zero. Exit.." + exit $ret + fi + +} + +do_rmtmp() { + log " Removing tmp" + rm -rf bitbake.lock pseudodone conf/sanity_info cache tmp +} +do_rmsstate () { + log " Removing sstate-cache" + rm -rf sstate-cache +} +do_sync () { + log " Syncing and dropping caches" + sync; sync + if [ $reqpass -eq 0 ]; then + sudo sh -c "echo 3 > /proc/sys/vm/drop_caches" + else + echo "$pass" | sudo -S sh -c "echo 3 > /proc/sys/vm/drop_caches" + echo + fi + sleep 3 +} + +write_results() { + echo -n "`uname -n`,$branch:$gitcommit,`git describe`," >> $globalres + for i in "${TIMES[@]}"; do + echo -n "$i," >> $globalres + done + for i in "${SIZES[@]}"; do + echo -n "$i," >> $globalres + done + echo >> $globalres + sed -i '$ s/,$//' $globalres +} + +#### + +# +# Test 1 +# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir (w/o rm_work and w/ rm_work) +# Pre: Downloaded sources, no sstate +# Steps: +# Part1: +# - fetchall +# - clean build dir +# - time bitbake core-image-sato +# - collect data +# Part2: +# - bitbake virtual/kernel -c cleansstate +# - time bitbake virtual/kernel +# Part3: +# - add INHERIT to local.conf +# - clean build dir +# - build +# - report size, remove INHERIT + +test1_p1 () { + log "Running Test 1, part 1/3: Measure wall clock of bitbake $IMAGE and size of tmp/ dir" + bbnotime $IMAGE -c fetchall + do_rmtmp + do_rmsstate + do_sync + bbtime $IMAGE + s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'` + SIZES[(( size_count++ ))]="$s" + log "SIZE of tmp dir is: $s" + log "Buildstats are saved in $OUTDIR/buildstats-test1" + mv tmp/buildstats $OUTDIR/buildstats-test1 +} + + +test1_p2 () { + log "Running Test 1, part 2/3: bitbake virtual/kernel -c cleansstate and time bitbake virtual/kernel" + bbnotime virtual/kernel -c cleansstate + do_sync + bbtime virtual/kernel +} + +test1_p3 () { + log "Running Test 1, part 3/3: Build $IMAGE w/o sstate and report size of tmp/dir with rm_work enabled" + echo "INHERIT += \"rm_work\"" >> conf/local.conf + do_rmtmp + do_rmsstate + do_sync + bbtime $IMAGE + sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf + s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'` + SIZES[(( size_count++ ))]="$s" + log "SIZE of tmp dir is: $s" + log "Buildstats are saved in $OUTDIR/buildstats-test13" + mv tmp/buildstats $OUTDIR/buildstats-test13 +} + + +# +# Test 2 +# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir +# Pre: populated sstate cache + +test2 () { + # Assuming test 1 has run + log "Running Test 2: Measure wall clock of bitbake $IMAGE -c rootfs with sstate" + do_rmtmp + do_sync + bbtime $IMAGE -c rootfs +} + + +# Test 3 +# parsing time metrics +# +# Start with +# i) "rm -rf tmp/cache; time bitbake -p" +# ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p" +# iii) "time bitbake -p" + + +test3 () { + log "Running Test 3: Parsing time metrics (bitbake -p)" + log " Removing tmp/cache && cache" + rm -rf tmp/cache cache + bbtime -p + log " Removing tmp/cache/default-glibc/" + rm -rf tmp/cache/default-glibc/ + bbtime -p + bbtime -p +} + +# +# Test 4 - eSDK +# Measure: eSDK size and installation time +test4 () { + log "Running Test 4: eSDK size and installation time" + bbnotime $IMAGE -c do_populate_sdk_ext + + esdk_installer=(tmp/deploy/sdk/*-toolchain-ext-*.sh) + + if [ ${#esdk_installer[*]} -eq 1 ]; then + s=$((`stat -c %s "$esdk_installer"` / 1024)) + SIZES[(( size_count++ ))]="$s" + log "Download SIZE of eSDK is: $s kB" + + do_sync + time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy" + + s=$((`du -sb "tmp/esdk-deploy" | cut -f1` / 1024)) + SIZES[(( size_count++ ))]="$s" + log "Install SIZE of eSDK is: $s kB" + else + log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0." + SIZES[(( size_count++ ))]="0" + TIMES[(( time_count++ ))]="0" + fi + +} + + +# RUN! + +test1_p1 +test1_p2 +test1_p3 +test2 +test3 +test4 + +# if we got til here write to global results +write_results + +log "All done, cleaning up..." + +do_rmtmp +do_rmsstate diff --git a/scripts/contrib/ddimage b/scripts/contrib/ddimage new file mode 100755 index 0000000..a503f11 --- /dev/null +++ b/scripts/contrib/ddimage @@ -0,0 +1,104 @@ +#!/bin/sh + +# Default to avoiding the first two disks on typical Linux and Mac OS installs +# Better safe than sorry :-) +BLACKLIST_DEVICES="/dev/sda /dev/sdb /dev/disk1 /dev/disk2" + +# 1MB blocksize +BLOCKSIZE=1048576 + +usage() { + echo "Usage: $(basename $0) IMAGE DEVICE" +} + +image_details() { + IMG=$1 + echo "Image details" + echo "=============" + echo " image: $(basename $IMG)" + # stat format is different on Mac OS and Linux + if [ "$(uname)" = "Darwin" ]; then + echo " size: $(stat -L -f '%z bytes' $IMG)" + echo " modified: $(stat -L -f '%Sm' $IMG)" + else + echo " size: $(stat -L -c '%s bytes' $IMG)" + echo " modified: $(stat -L -c '%y' $IMG)" + fi + echo " type: $(file -L -b $IMG)" + echo "" +} + +device_details() { + DEV=$1 + BLOCK_SIZE=512 + + echo "Device details" + echo "==============" + + # Collect disk info using diskutil on Mac OS + if [ "$(uname)" = "Darwin" ]; then + diskutil info $DEVICE | egrep "(Device Node|Media Name|Total Size)" + return + fi + + # Default / Linux information collection + echo " device: $DEVICE" + if [ -f "/sys/class/block/$DEV/device/vendor" ]; then + echo " vendor: $(cat /sys/class/block/$DEV/device/vendor)" + else + echo " vendor: UNKOWN" + fi + if [ -f "/sys/class/block/$DEV/device/model" ]; then + echo " model: $(cat /sys/class/block/$DEV/device/model)" + else + echo " model: UNKNOWN" + fi + if [ -f "/sys/class/block/$DEV/size" ]; then + echo " size: $(($(cat /sys/class/block/$DEV/size) * $BLOCK_SIZE)) bytes" + else + echo " size: UNKNOWN" + fi + echo "" +} + +if [ $# -ne 2 ]; then + usage + exit 1 +fi + +IMAGE=$1 +DEVICE=$2 + +if [ ! -e "$IMAGE" ]; then + echo "ERROR: Image $IMAGE does not exist" + usage + exit 1 +fi + + +for i in ${BLACKLIST_DEVICES}; do + if [ "$i" = "$DEVICE" ]; then + echo "ERROR: Device $DEVICE is blacklisted" + exit 1 + fi +done + +if [ ! -w "$DEVICE" ]; then + echo "ERROR: Device $DEVICE does not exist or is not writable" + usage + exit 1 +fi + +image_details $IMAGE +device_details $(basename $DEVICE) + +printf "Write $IMAGE to $DEVICE [y/N]? " +read RESPONSE +if [ "$RESPONSE" != "y" ]; then + echo "Write aborted" + exit 0 +fi + +echo "Writing image..." +dd if="$IMAGE" of="$DEVICE" bs="$BLOCKSIZE" +sync diff --git a/scripts/contrib/devtool-stress.py b/scripts/contrib/devtool-stress.py new file mode 100755 index 0000000..8cf92ca --- /dev/null +++ b/scripts/contrib/devtool-stress.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python + +# devtool stress tester +# +# Written by: Paul Eggleton <paul.eggleton@linux.intel.com> +# +# Copyright 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import sys +import os +import os.path +import subprocess +import re +import argparse +import logging +import tempfile +import shutil +import signal +import fnmatch + +scripts_lib_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'lib')) +sys.path.insert(0, scripts_lib_path) +import scriptutils +import argparse_oe +logger = scriptutils.logger_create('devtool-stress') + +def select_recipes(args): + import bb.tinfoil + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(False) + + pkg_pn = tinfoil.cooker.recipecache.pkg_pn + (latest_versions, preferred_versions) = bb.providers.findProviders(tinfoil.config_data, tinfoil.cooker.recipecache, pkg_pn) + + skip_classes = args.skip_classes.split(',') + + recipelist = [] + for pn in sorted(pkg_pn): + pref = preferred_versions[pn] + inherits = [os.path.splitext(os.path.basename(f))[0] for f in tinfoil.cooker.recipecache.inherits[pref[1]]] + for cls in skip_classes: + if cls in inherits: + break + else: + recipelist.append(pn) + + tinfoil.shutdown() + + resume_from = args.resume_from + if resume_from: + if not resume_from in recipelist: + print('%s is not a testable recipe' % resume_from) + return 1 + if args.only: + only = args.only.split(',') + for onlyitem in only: + for pn in recipelist: + if fnmatch.fnmatch(pn, onlyitem): + break + else: + print('%s does not match any testable recipe' % onlyitem) + return 1 + else: + only = None + if args.skip: + skip = args.skip.split(',') + else: + skip = [] + + recipes = [] + for pn in recipelist: + if resume_from: + if pn == resume_from: + resume_from = None + else: + continue + + if args.only: + for item in only: + if fnmatch.fnmatch(pn, item): + break + else: + continue + + skipit = False + for item in skip: + if fnmatch.fnmatch(pn, item): + skipit = True + if skipit: + continue + + recipes.append(pn) + + return recipes + + +def stress_extract(args): + import bb.process + + recipes = select_recipes(args) + + failures = 0 + tmpdir = tempfile.mkdtemp() + os.setpgrp() + try: + for pn in recipes: + sys.stdout.write('Testing %s ' % (pn + ' ').ljust(40, '.')) + sys.stdout.flush() + failed = False + + srctree = os.path.join(tmpdir, pn) + try: + bb.process.run('devtool extract %s %s' % (pn, srctree)) + except bb.process.CmdError as exc: + failed = True + with open('stress_%s_extract.log' % pn, 'w') as f: + f.write(str(exc)) + + if os.path.exists(srctree): + shutil.rmtree(srctree) + + if failed: + print('failed') + failures += 1 + else: + print('ok') + except KeyboardInterrupt: + # We want any child processes killed. This is crude, but effective. + os.killpg(0, signal.SIGTERM) + + if failures: + return 1 + else: + return 0 + + +def stress_modify(args): + import bb.process + + recipes = select_recipes(args) + + failures = 0 + tmpdir = tempfile.mkdtemp() + os.setpgrp() + try: + for pn in recipes: + sys.stdout.write('Testing %s ' % (pn + ' ').ljust(40, '.')) + sys.stdout.flush() + failed = False + reset = True + + srctree = os.path.join(tmpdir, pn) + try: + bb.process.run('devtool modify -x %s %s' % (pn, srctree)) + except bb.process.CmdError as exc: + with open('stress_%s_modify.log' % pn, 'w') as f: + f.write(str(exc)) + failed = 'modify' + reset = False + + if not failed: + try: + bb.process.run('bitbake -c install %s' % pn) + except bb.process.CmdError as exc: + with open('stress_%s_install.log' % pn, 'w') as f: + f.write(str(exc)) + failed = 'build' + if reset: + try: + bb.process.run('devtool reset %s' % pn) + except bb.process.CmdError as exc: + print('devtool reset failed: %s' % str(exc)) + break + + if os.path.exists(srctree): + shutil.rmtree(srctree) + + if failed: + print('failed (%s)' % failed) + failures += 1 + else: + print('ok') + except KeyboardInterrupt: + # We want any child processes killed. This is crude, but effective. + os.killpg(0, signal.SIGTERM) + + if failures: + return 1 + else: + return 0 + + +def main(): + parser = argparse_oe.ArgumentParser(description="devtool stress tester", + epilog="Use %(prog)s <subcommand> --help to get help on a specific command") + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-r', '--resume-from', help='Resume from specified recipe', metavar='PN') + parser.add_argument('-o', '--only', help='Only test specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST') + parser.add_argument('-s', '--skip', help='Skip specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST') + parser.add_argument('-c', '--skip-classes', help='Skip recipes inheriting specified classes (comma-separated) - default %(default)s', metavar='CLASSLIST', default='native,nativesdk,cross,cross-canadian,image,populate_sdk,meta,packagegroup') + subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') + + parser_modify = subparsers.add_parser('modify', + help='Run "devtool modify" followed by a build with bitbake on matching recipes', + description='Runs "devtool modify" followed by a build with bitbake on matching recipes') + parser_modify.set_defaults(func=stress_modify) + + parser_extract = subparsers.add_parser('extract', + help='Run "devtool extract" on matching recipes', + description='Runs "devtool extract" on matching recipes') + parser_extract.set_defaults(func=stress_extract) + + args = parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + + import scriptpath + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + return 1 + logger.debug('Found bitbake path: %s' % bitbakepath) + + ret = args.func(args) + +if __name__ == "__main__": + main() diff --git a/scripts/contrib/dialog-power-control b/scripts/contrib/dialog-power-control new file mode 100755 index 0000000..7550ea5 --- /dev/null +++ b/scripts/contrib/dialog-power-control @@ -0,0 +1,53 @@ +#!/bin/sh +# +# Simple script to show a manual power prompt for when you want to use +# automated hardware testing with testimage.bbclass but you don't have a +# web-enabled power strip or similar to do the power on/off/cycle. +# +# You can enable it by enabling testimage (see the Yocto Project +# Development manual "Performing Automated Runtime Testing" section) +# and setting the following in your local.conf: +# +# TEST_POWERCONTROL_CMD = "${COREBASE}/scripts/contrib/dialog-power-control" +# + +PROMPT="" +while true; do + case $1 in + on) + PROMPT="Please turn device power on";; + off) + PROMPT="Please turn device power off";; + cycle) + PROMPT="Please click Done, then turn the device power off then on";; + "") + break;; + esac + shift +done + +if [ "$PROMPT" = "" ] ; then + echo "ERROR: no power action specified on command line" + exit 2 +fi + +if [ "`which kdialog 2>/dev/null`" != "" ] ; then + DIALOGUTIL="kdialog" +elif [ "`which zenity 2>/dev/null`" != "" ] ; then + DIALOGUTIL="zenity" +else + echo "ERROR: couldn't find program to display a message, install kdialog or zenity" + exit 3 +fi + +if [ "$DIALOGUTIL" = "kdialog" ] ; then + kdialog --yesno "$PROMPT" --title "TestImage Power Control" --yes-label "Done" --no-label "Cancel test" +elif [ "$DIALOGUTIL" = "zenity" ] ; then + zenity --question --text="$PROMPT" --title="TestImage Power Control" --ok-label="Done" --cancel-label="Cancel test" +fi + +if [ "$?" != "0" ] ; then + echo "User cancelled test at power prompt" + exit 1 +fi + diff --git a/scripts/contrib/documentation-audit.sh b/scripts/contrib/documentation-audit.sh new file mode 100755 index 0000000..2144aac --- /dev/null +++ b/scripts/contrib/documentation-audit.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# +# Perform an audit of which packages provide documentation and which +# are missing -doc packages. +# +# Setup requirements: be sure to be building for MACHINE=qemux86. Run +# this script after source'ing the build environment script, so you're +# running it from build/ directory. +# +# Maintainer: Scott Garman <scott.a.garman@intel.com> + +REPORT_DOC_SIMPLE="documentation_exists.txt" +REPORT_DOC_DETAIL="documentation_exists_detail.txt" +REPORT_MISSING_SIMPLE="documentation_missing.txt" +REPORT_MISSING_DETAIL="documentation_missing_detail.txt" +REPORT_BUILD_ERRORS="build_errors.txt" + +rm -rf $REPORT_DOC_SIMPLE $REPORT_DOC_DETAIL $REPORT_MISSING_SIMPLE $REPORT_MISSING_DETAIL + +BITBAKE=`which bitbake` +if [ -z "$BITBAKE" ]; then + echo "Error: bitbake command not found." + echo "Did you forget to source the build environment script?" + exit 1 +fi + +echo "REMINDER: you need to build for MACHINE=qemux86 or you won't get useful results" +echo "REMINDER: you need to set LICENSE_FLAGS_WHITELIST appropriately in local.conf or " +echo " you'll get false positives. For example, LICENSE_FLAGS_WHITELIST = \"Commercial\"" + +for pkg in `bitbake -s | awk '{ print \$1 }'`; do + if [[ "$pkg" == "Loading" || "$pkg" == "Loaded" || + "$pkg" == "Recipe" || + "$pkg" == "Parsing" || "$pkg" == "Package" || + "$pkg" == "NOTE:" || "$pkg" == "WARNING:" || + "$pkg" == "done." || "$pkg" == "===========" ]] + then + # Skip initial bitbake output + continue + fi + if [[ "$pkg" =~ -native$ || "$pkg" =~ -nativesdk$ || + "$pkg" =~ -cross-canadian ]]; then + # Skip native/nativesdk/cross-canadian recipes + continue + fi + if [[ "$pkg" =~ ^meta- || "$pkg" =~ ^packagegroup- || "$pkg" =~ -image ]]; then + # Skip meta, task and image recipes + continue + fi + if [[ "$pkg" =~ ^glibc- || "$pkg" =~ ^libiconv$ || + "$pkg" =~ -toolchain$ || "$pkg" =~ ^package-index$ || + "$pkg" =~ ^linux- || "$pkg" =~ ^adt-installer$ || + "$pkg" =~ ^eds-tools$ || "$pkg" =~ ^external-python-tarball$ || + "$pkg" =~ ^qt4-embedded$ || "$pkg" =~ ^qt-mobility ]]; then + # Skip glibc, libiconv, -toolchain, and other recipes known + # to cause build conflicts or trigger false positives. + continue + fi + + echo "Building package $pkg..." + bitbake $pkg > /dev/null + if [ $? -ne 0 ]; then + echo "There was an error building package $pkg" >> "$REPORT_MISSING_DETAIL" + echo "$pkg" >> $REPORT_BUILD_ERRORS + + # Do not skip the remaining tests, as sometimes the + # exit status is 1 due to QA errors, and we can still + # perform the -doc checks. + fi + + echo "$pkg built successfully, checking for a documentation package..." + WORKDIR=`bitbake -e $pkg | grep ^WORKDIR | awk -F '=' '{ print \$2 }' | awk -F '"' '{ print \$2 }'` + FIND_DOC_PKG=`find $WORKDIR/packages-split/*-doc -maxdepth 0 -type d` + if [ -z "$FIND_DOC_PKG" ]; then + # No -doc package was generated: + echo "No -doc package: $pkg" >> "$REPORT_MISSING_DETAIL" + echo "$pkg" >> $REPORT_MISSING_SIMPLE + continue + fi + + FIND_DOC_FILES=`find $FIND_DOC_PKG -type f` + if [ -z "$FIND_DOC_FILES" ]; then + # No files shipped with the -doc package: + echo "No files shipped with the -doc package: $pkg" >> "$REPORT_MISSING_DETAIL" + echo "$pkg" >> $REPORT_MISSING_SIMPLE + continue + fi + + echo "Documentation shipped with $pkg:" >> "$REPORT_DOC_DETAIL" + echo "$FIND_DOC_FILES" >> "$REPORT_DOC_DETAIL" + echo "" >> "$REPORT_DOC_DETAIL" + + echo "$pkg" >> "$REPORT_DOC_SIMPLE" +done diff --git a/scripts/contrib/graph-tool b/scripts/contrib/graph-tool new file mode 100755 index 0000000..6dc7d33 --- /dev/null +++ b/scripts/contrib/graph-tool @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +# Simple graph query utility +# useful for getting answers from .dot files produced by bitbake -g +# +# Written by: Paul Eggleton <paul.eggleton@linux.intel.com> +# +# Copyright 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import sys + +def get_path_networkx(dotfile, fromnode, tonode): + try: + import networkx + except ImportError: + print('ERROR: Please install the networkx python module') + sys.exit(1) + + graph = networkx.DiGraph(networkx.read_dot(dotfile)) + + def node_missing(node): + import difflib + close_matches = difflib.get_close_matches(node, graph.nodes(), cutoff=0.7) + if close_matches: + print('ERROR: no node "%s" in graph. Close matches:\n %s' % (node, '\n '.join(close_matches))) + sys.exit(1) + + if not fromnode in graph: + node_missing(fromnode) + if not tonode in graph: + node_missing(tonode) + return networkx.all_simple_paths(graph, source=fromnode, target=tonode) + + +def find_paths(args, usage): + if len(args) < 3: + usage() + sys.exit(1) + + fromnode = args[1] + tonode = args[2] + paths = list(get_path_networkx(args[0], fromnode, tonode)) + if paths: + for path in paths: + print ' -> '.join(path) + else: + print("ERROR: no path from %s to %s in graph" % (fromnode, tonode)) + sys.exit(1) + +def main(): + import optparse + parser = optparse.OptionParser( + usage = '''%prog [options] <command> <arguments> + +Available commands: + find-paths <dotfile> <from> <to> + Find all of the paths between two nodes in a dot graph''') + + #parser.add_option("-d", "--debug", + # help = "Report all SRCREV values, not just ones where AUTOREV has been used", + # action="store_true", dest="debug", default=False) + + options, args = parser.parse_args(sys.argv) + args = args[1:] + + if len(args) < 1: + parser.print_help() + sys.exit(1) + + if args[0] == "find-paths": + find_paths(args[1:], parser.print_help) + else: + parser.print_help() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/contrib/list-packageconfig-flags.py b/scripts/contrib/list-packageconfig-flags.py new file mode 100755 index 0000000..2f3b8b0 --- /dev/null +++ b/scripts/contrib/list-packageconfig-flags.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation. +# +# Copyright (C) 2013 Wind River Systems, Inc. +# Copyright (C) 2014 Intel Corporation +# +# - list available recipes which have PACKAGECONFIG flags +# - list available PACKAGECONFIG flags and all affected recipes +# - list all recipes and PACKAGECONFIG information + +import sys +import optparse +import os + + +scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) +lib_path = os.path.abspath(scripts_path + '/../lib') +sys.path = sys.path + [lib_path] + +import scriptpath + +# For importing the following modules +bitbakepath = scriptpath.add_bitbake_lib_path() +if not bitbakepath: + sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") + sys.exit(1) + +import bb.cache +import bb.cooker +import bb.providers +import bb.tinfoil + +def get_fnlist(bbhandler, pkg_pn, preferred): + ''' Get all recipe file names ''' + if preferred: + (latest_versions, preferred_versions) = bb.providers.findProviders(bbhandler.config_data, bbhandler.cooker.recipecache, pkg_pn) + + fn_list = [] + for pn in sorted(pkg_pn): + if preferred: + fn_list.append(preferred_versions[pn][1]) + else: + fn_list.extend(pkg_pn[pn]) + + return fn_list + +def get_recipesdata(bbhandler, preferred): + ''' Get data of all available recipes which have PACKAGECONFIG flags ''' + pkg_pn = bbhandler.cooker.recipecache.pkg_pn + + data_dict = {} + for fn in get_fnlist(bbhandler, pkg_pn, preferred): + data = bb.cache.Cache.loadDataFull(fn, bbhandler.cooker.collection.get_file_appends(fn), bbhandler.config_data) + flags = data.getVarFlags("PACKAGECONFIG") + flags.pop('doc', None) + if flags: + data_dict[fn] = data + + return data_dict + +def collect_pkgs(data_dict): + ''' Collect available pkgs in which have PACKAGECONFIG flags ''' + # pkg_dict = {'pkg1': ['flag1', 'flag2',...]} + pkg_dict = {} + for fn in data_dict: + pkgconfigflags = data_dict[fn].getVarFlags("PACKAGECONFIG") + pkgconfigflags.pop('doc', None) + pkgname = data_dict[fn].getVar("P", True) + pkg_dict[pkgname] = sorted(pkgconfigflags.keys()) + + return pkg_dict + +def collect_flags(pkg_dict): + ''' Collect available PACKAGECONFIG flags and all affected pkgs ''' + # flag_dict = {'flag': ['pkg1', 'pkg2',...]} + flag_dict = {} + for pkgname, flaglist in pkg_dict.iteritems(): + for flag in flaglist: + if flag in flag_dict: + flag_dict[flag].append(pkgname) + else: + flag_dict[flag] = [pkgname] + + return flag_dict + +def display_pkgs(pkg_dict): + ''' Display available pkgs which have PACKAGECONFIG flags ''' + pkgname_len = len("RECIPE NAME") + 1 + for pkgname in pkg_dict: + if pkgname_len < len(pkgname): + pkgname_len = len(pkgname) + pkgname_len += 1 + + header = '%-*s%s' % (pkgname_len, str("RECIPE NAME"), str("PACKAGECONFIG FLAGS")) + print header + print str("").ljust(len(header), '=') + for pkgname in sorted(pkg_dict): + print('%-*s%s' % (pkgname_len, pkgname, ' '.join(pkg_dict[pkgname]))) + + +def display_flags(flag_dict): + ''' Display available PACKAGECONFIG flags and all affected pkgs ''' + flag_len = len("PACKAGECONFIG FLAG") + 5 + + header = '%-*s%s' % (flag_len, str("PACKAGECONFIG FLAG"), str("RECIPE NAMES")) + print header + print str("").ljust(len(header), '=') + + for flag in sorted(flag_dict): + print('%-*s%s' % (flag_len, flag, ' '.join(sorted(flag_dict[flag])))) + +def display_all(data_dict): + ''' Display all pkgs and PACKAGECONFIG information ''' + print str("").ljust(50, '=') + for fn in data_dict: + print('%s' % data_dict[fn].getVar("P", True)) + print fn + packageconfig = data_dict[fn].getVar("PACKAGECONFIG", True) or '' + if packageconfig.strip() == '': + packageconfig = 'None' + print('PACKAGECONFIG %s' % packageconfig) + + for flag,flag_val in data_dict[fn].getVarFlags("PACKAGECONFIG").iteritems(): + if flag == "doc": + continue + print('PACKAGECONFIG[%s] %s' % (flag, flag_val)) + print '' + +def main(): + pkg_dict = {} + flag_dict = {} + + # Collect and validate input + parser = optparse.OptionParser( + description = "Lists recipes and PACKAGECONFIG flags. Without -a or -f, recipes and their available PACKAGECONFIG flags are listed.", + usage = """ + %prog [options]""") + + parser.add_option("-f", "--flags", + help = "list available PACKAGECONFIG flags and affected recipes", + action="store_const", dest="listtype", const="flags", default="recipes") + parser.add_option("-a", "--all", + help = "list all recipes and PACKAGECONFIG information", + action="store_const", dest="listtype", const="all") + parser.add_option("-p", "--preferred-only", + help = "where multiple recipe versions are available, list only the preferred version", + action="store_true", dest="preferred", default=False) + + options, args = parser.parse_args(sys.argv) + + bbhandler = bb.tinfoil.Tinfoil() + bbhandler.prepare() + print("Gathering recipe data...") + data_dict = get_recipesdata(bbhandler, options.preferred) + + if options.listtype == 'flags': + pkg_dict = collect_pkgs(data_dict) + flag_dict = collect_flags(pkg_dict) + display_flags(flag_dict) + elif options.listtype == 'recipes': + pkg_dict = collect_pkgs(data_dict) + display_pkgs(pkg_dict) + elif options.listtype == 'all': + display_all(data_dict) + +if __name__ == "__main__": + main() diff --git a/scripts/contrib/mkefidisk.sh b/scripts/contrib/mkefidisk.sh new file mode 100755 index 0000000..d8db3c0 --- /dev/null +++ b/scripts/contrib/mkefidisk.sh @@ -0,0 +1,459 @@ +#!/bin/sh +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See +# the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +LANG=C + +# Set to 1 to enable additional output +DEBUG=0 +OUT="/dev/null" + +# +# Defaults +# +# 20 Mb for the boot partition +BOOT_SIZE=20 +# 5% for swap +SWAP_RATIO=5 + +# Cleanup after die() +cleanup() { + debug "Syncing and unmounting devices" + # Unmount anything we mounted + unmount $ROOTFS_MNT || error "Failed to unmount $ROOTFS_MNT" + unmount $BOOTFS_MNT || error "Failed to unmount $BOOTFS_MNT" + unmount $HDDIMG_ROOTFS_MNT || error "Failed to unmount $HDDIMG_ROOTFS_MNT" + unmount $HDDIMG_MNT || error "Failed to unmount $HDDIMG_MNT" + + # Remove the TMPDIR + debug "Removing temporary files" + if [ -d "$TMPDIR" ]; then + rm -rf $TMPDIR || error "Failed to remove $TMPDIR" + fi +} + +trap 'die "Signal Received, Aborting..."' HUP INT TERM + +# Logging routines +WARNINGS=0 +ERRORS=0 +CLEAR="$(tput sgr0)" +INFO="$(tput bold)" +RED="$(tput setaf 1)$(tput bold)" +GREEN="$(tput setaf 2)$(tput bold)" +YELLOW="$(tput setaf 3)$(tput bold)" +info() { + echo "${INFO}$1${CLEAR}" +} +error() { + ERRORS=$((ERRORS+1)) + echo "${RED}$1${CLEAR}" +} +warn() { + WARNINGS=$((WARNINGS+1)) + echo "${YELLOW}$1${CLEAR}" +} +success() { + echo "${GREEN}$1${CLEAR}" +} +die() { + error "$1" + cleanup + exit 1 +} +debug() { + if [ $DEBUG -eq 1 ]; then + echo "$1" + fi +} + +usage() { + echo "Usage: $(basename $0) [-v] DEVICE HDDIMG TARGET_DEVICE" + echo " -v: Verbose debug" + echo " DEVICE: The device to write the image to, e.g. /dev/sdh" + echo " HDDIMG: The hddimg file to generate the efi disk from" + echo " TARGET_DEVICE: The device the target will boot from, e.g. /dev/mmcblk0" +} + +image_details() { + IMG=$1 + info "Image details" + echo " image: $(stat --printf '%N\n' $IMG)" + echo " size: $(stat -L --printf '%s bytes\n' $IMG)" + echo " modified: $(stat -L --printf '%y\n' $IMG)" + echo " type: $(file -L -b $IMG)" + echo "" +} + +device_details() { + DEV=$1 + BLOCK_SIZE=512 + + info "Device details" + echo " device: $DEVICE" + if [ -f "/sys/class/block/$DEV/device/vendor" ]; then + echo " vendor: $(cat /sys/class/block/$DEV/device/vendor)" + else + echo " vendor: UNKOWN" + fi + if [ -f "/sys/class/block/$DEV/device/model" ]; then + echo " model: $(cat /sys/class/block/$DEV/device/model)" + else + echo " model: UNKNOWN" + fi + if [ -f "/sys/class/block/$DEV/size" ]; then + echo " size: $(($(cat /sys/class/block/$DEV/size) * $BLOCK_SIZE)) bytes" + else + echo " size: UNKNOWN" + fi + echo "" +} + +unmount_device() { + grep -q $DEVICE /proc/mounts + if [ $? -eq 0 ]; then + warn "$DEVICE listed in /proc/mounts, attempting to unmount" + umount $DEVICE* 2>/dev/null + return $? + fi + return 0 +} + +unmount() { + if [ "$1" = "" ] ; then + return 0 + fi + grep -q $1 /proc/mounts + if [ $? -eq 0 ]; then + debug "Unmounting $1" + umount $1 + return $? + fi + return 0 +} + +# +# Parse and validate arguments +# +if [ $# -lt 3 ] || [ $# -gt 4 ]; then + if [ $# -eq 1 ]; then + AVAILABLE_DISK=`lsblk | grep "disk" | cut -f 1 -d " "` + X=0 + for disk in `echo $AVAILABLE_DISK`; do + mounted=`lsblk /dev/$disk | awk {'print $7'} | sed "s/MOUNTPOINT//"` + if [ -z "$mounted" ]; then + UNMOUNTED_AVAILABLES="$UNMOUNTED_AVAILABLES /dev/$disk" + info "$X - /dev/$disk" + X=`expr $X + 1` + fi + done + if [ $X -eq 0 ]; then + die "No unmounted device found." + fi + read -p "Choose unmounted device number: " DISK_NUMBER + X=0 + for line in `echo $UNMOUNTED_AVAILABLES`; do + if [ $DISK_NUMBER -eq $X ]; then + DISK_TO_BE_FLASHED=$line + break + else + X=`expr $X + 1` + fi + done + if [ -z "$DISK_TO_BE_FLASHED" ]; then + die "Option \"$DISK_NUMBER\" is invalid. Choose a valid option" + else + if [ -z `echo $DISK_TO_BE_FLASHED | grep "mmc"` ]; then + TARGET_TO_BE_BOOT="/dev/sda" + else + TARGET_TO_BE_BOOT="/dev/mmcblk0" + fi + fi + echo "" + echo "Choose a name of the device that will be boot from" + echo -n "Recommended name is: " + info "$TARGET_TO_BE_BOOT" + read -p "Is target device okay? [y/N]: " RESPONSE + if [ "$RESPONSE" != "y" ]; then + read -p "Choose target device name: " TARGET_TO_BE_BOOT + fi + echo "" + if [ -z "$TARGET_TO_BE_BOOT" ]; then + die "Error: choose a valid target name" + fi + else + usage + exit 1 + fi +fi + +if [ "$1" = "-v" ]; then + DEBUG=1 + OUT="1" + shift +fi + +if [ -z "$AVAILABLE_DISK" ]; then + DEVICE=$1 + HDDIMG=$2 + TARGET_DEVICE=$3 +else + DEVICE=$DISK_TO_BE_FLASHED + HDDIMG=$1 + TARGET_DEVICE=$TARGET_TO_BE_BOOT +fi + +LINK=$(readlink $DEVICE) +if [ $? -eq 0 ]; then + DEVICE="$LINK" +fi + +if [ ! -w "$DEVICE" ]; then + usage + if [ ! -e "${DEVICE}" ] ; then + die "Device $DEVICE cannot be found" + else + die "Device $DEVICE is not writable (need to run under sudo?)" + fi +fi + +if [ ! -e "$HDDIMG" ]; then + usage + die "HDDIMG $HDDIMG does not exist" +fi + +# +# Ensure the hddimg is not mounted +# +unmount "$HDDIMG" || die "Failed to unmount $HDDIMG" + +# +# Check if any $DEVICE partitions are mounted +# +unmount_device || die "Failed to unmount $DEVICE" + +# +# Confirm device with user +# +image_details $HDDIMG +device_details $(basename $DEVICE) +echo -n "${INFO}Prepare EFI image on $DEVICE [y/N]?${CLEAR} " +read RESPONSE +if [ "$RESPONSE" != "y" ]; then + echo "Image creation aborted" + exit 0 +fi + + +# +# Prepare the temporary working space +# +TMPDIR=$(mktemp -d mkefidisk-XXX) || die "Failed to create temporary mounting directory." +HDDIMG_MNT=$TMPDIR/hddimg +HDDIMG_ROOTFS_MNT=$TMPDIR/hddimg_rootfs +ROOTFS_MNT=$TMPDIR/rootfs +BOOTFS_MNT=$TMPDIR/bootfs +mkdir $HDDIMG_MNT || die "Failed to create $HDDIMG_MNT" +mkdir $HDDIMG_ROOTFS_MNT || die "Failed to create $HDDIMG_ROOTFS_MNT" +mkdir $ROOTFS_MNT || die "Failed to create $ROOTFS_MNT" +mkdir $BOOTFS_MNT || die "Failed to create $BOOTFS_MNT" + + +# +# Partition $DEVICE +# +DEVICE_SIZE=$(parted -s $DEVICE unit mb print | grep ^Disk | cut -d" " -f 3 | sed -e "s/MB//") +# If the device size is not reported there may not be a valid label +if [ "$DEVICE_SIZE" = "" ] ; then + parted -s $DEVICE mklabel msdos || die "Failed to create MSDOS partition table" + DEVICE_SIZE=$(parted -s $DEVICE unit mb print | grep ^Disk | cut -d" " -f 3 | sed -e "s/MB//") +fi +SWAP_SIZE=$((DEVICE_SIZE*SWAP_RATIO/100)) +ROOTFS_SIZE=$((DEVICE_SIZE-BOOT_SIZE-SWAP_SIZE)) +ROOTFS_START=$((BOOT_SIZE)) +ROOTFS_END=$((ROOTFS_START+ROOTFS_SIZE)) +SWAP_START=$((ROOTFS_END)) + +# MMC devices use a partition prefix character 'p' +PART_PREFIX="" +if [ ! "${DEVICE#/dev/mmcblk}" = "${DEVICE}" ] || [ ! "${DEVICE#/dev/loop}" = "${DEVICE}" ]; then + PART_PREFIX="p" +fi +BOOTFS=$DEVICE${PART_PREFIX}1 +ROOTFS=$DEVICE${PART_PREFIX}2 +SWAP=$DEVICE${PART_PREFIX}3 + +TARGET_PART_PREFIX="" +if [ ! "${TARGET_DEVICE#/dev/mmcblk}" = "${TARGET_DEVICE}" ]; then + TARGET_PART_PREFIX="p" +fi +TARGET_ROOTFS=$TARGET_DEVICE${TARGET_PART_PREFIX}2 +TARGET_SWAP=$TARGET_DEVICE${TARGET_PART_PREFIX}3 + +echo "" +info "Boot partition size: $BOOT_SIZE MB ($BOOTFS)" +info "ROOTFS partition size: $ROOTFS_SIZE MB ($ROOTFS)" +info "Swap partition size: $SWAP_SIZE MB ($SWAP)" +echo "" + +# Use MSDOS by default as GPT cannot be reliably distributed in disk image form +# as it requires the backup table to be on the last block of the device, which +# of course varies from device to device. + +info "Partitioning installation media ($DEVICE)" + +debug "Deleting partition table on $DEVICE" +dd if=/dev/zero of=$DEVICE bs=512 count=2 >$OUT 2>&1 || die "Failed to zero beginning of $DEVICE" + +debug "Creating new partition table (MSDOS) on $DEVICE" +parted -s $DEVICE mklabel msdos >$OUT 2>&1 || die "Failed to create MSDOS partition table" + +debug "Creating boot partition on $BOOTFS" +parted -s $DEVICE mkpart primary 0% $BOOT_SIZE >$OUT 2>&1 || die "Failed to create BOOT partition" + +debug "Enabling boot flag on $BOOTFS" +parted -s $DEVICE set 1 boot on >$OUT 2>&1 || die "Failed to enable boot flag" + +debug "Creating ROOTFS partition on $ROOTFS" +parted -s $DEVICE mkpart primary $ROOTFS_START $ROOTFS_END >$OUT 2>&1 || die "Failed to create ROOTFS partition" + +debug "Creating swap partition on $SWAP" +parted -s $DEVICE mkpart primary $SWAP_START 100% >$OUT 2>&1 || die "Failed to create SWAP partition" + +if [ $DEBUG -eq 1 ]; then + parted -s $DEVICE print +fi + + +# +# Check if any $DEVICE partitions are mounted after partitioning +# +unmount_device || die "Failed to unmount $DEVICE partitions" + + +# +# Format $DEVICE partitions +# +info "Formatting partitions" +debug "Formatting $BOOTFS as vfat" +if [ ! "${DEVICE#/dev/loop}" = "${DEVICE}" ]; then + mkfs.vfat -I $BOOTFS -n "EFI" >$OUT 2>&1 || die "Failed to format $BOOTFS" +else + mkfs.vfat $BOOTFS -n "EFI" >$OUT 2>&1 || die "Failed to format $BOOTFS" +fi + +debug "Formatting $ROOTFS as ext3" +mkfs.ext3 -F $ROOTFS -L "ROOT" >$OUT 2>&1 || die "Failed to format $ROOTFS" + +debug "Formatting swap partition ($SWAP)" +mkswap $SWAP >$OUT 2>&1 || die "Failed to prepare swap" + + +# +# Installing to $DEVICE +# +debug "Mounting images and device in preparation for installation" +mount -o ro,loop $HDDIMG $HDDIMG_MNT >$OUT 2>&1 || error "Failed to mount $HDDIMG" +mount -o ro,loop $HDDIMG_MNT/rootfs.img $HDDIMG_ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount rootfs.img" +mount $ROOTFS $ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount $ROOTFS on $ROOTFS_MNT" +mount $BOOTFS $BOOTFS_MNT >$OUT 2>&1 || error "Failed to mount $BOOTFS on $BOOTFS_MNT" + +info "Preparing boot partition" +EFIDIR="$BOOTFS_MNT/EFI/BOOT" +cp $HDDIMG_MNT/vmlinuz $BOOTFS_MNT >$OUT 2>&1 || error "Failed to copy vmlinuz" +# Copy the efi loader and configs (booti*.efi and grub.cfg if it exists) +cp -r $HDDIMG_MNT/EFI $BOOTFS_MNT >$OUT 2>&1 || error "Failed to copy EFI dir" +# Silently ignore a missing gummiboot loader dir (we might just be a GRUB image) +cp -r $HDDIMG_MNT/loader $BOOTFS_MNT >$OUT 2>&1 + +# Update the boot loaders configurations for an installed image +# Remove any existing root= kernel parameters and: +# o Add a root= parameter with the target rootfs +# o Specify ro so fsck can be run during boot +# o Specify rootwait in case the target media is an asyncronous block device +# such as MMC or USB disks +# o Specify "quiet" to minimize boot time when using slow serial consoles + +# Look for a GRUB installation +GRUB_CFG="$EFIDIR/grub.cfg" +if [ -e "$GRUB_CFG" ]; then + info "Configuring GRUB" + # Delete the install entry + sed -i "/menuentry 'install'/,/^}/d" $GRUB_CFG + # Delete the initrd lines + sed -i "/initrd /d" $GRUB_CFG + # Delete any LABEL= strings + sed -i "s/ LABEL=[^ ]*/ /" $GRUB_CFG + + sed -i "s@ root=[^ ]*@ @" $GRUB_CFG + sed -i "s@vmlinuz @vmlinuz root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $GRUB_CFG +fi + +# Look for a gummiboot installation +GUMMI_ENTRIES="$BOOTFS_MNT/loader/entries" +GUMMI_CFG="$GUMMI_ENTRIES/boot.conf" +if [ -d "$GUMMI_ENTRIES" ]; then + info "Configuring Gummiboot" + # remove the install target if it exists + rm $GUMMI_ENTRIES/install.conf >$OUT 2>&1 + + if [ ! -e "$GUMMI_CFG" ]; then + echo "ERROR: $GUMMI_CFG not found" + fi + + sed -i "/initrd /d" $GUMMI_CFG + sed -i "s@ root=[^ ]*@ @" $GUMMI_CFG + sed -i "s@options *LABEL=boot @options LABEL=Boot root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $GUMMI_CFG +fi + +# Ensure we have at least one EFI bootloader configured +if [ ! -e $GRUB_CFG ] && [ ! -e $GUMMI_CFG ]; then + die "No EFI bootloader configuration found" +fi + + +info "Copying ROOTFS files (this may take a while)" +cp -a $HDDIMG_ROOTFS_MNT/* $ROOTFS_MNT >$OUT 2>&1 || die "Root FS copy failed" + +echo "$TARGET_SWAP swap swap defaults 0 0" >> $ROOTFS_MNT/etc/fstab + +# We dont want udev to mount our root device while we're booting... +if [ -d $ROOTFS_MNT/etc/udev/ ] ; then + echo "$TARGET_DEVICE" >> $ROOTFS_MNT/etc/udev/mount.blacklist +fi + +# Add startup.nsh script for automated boot +echo "fs0:\EFI\BOOT\bootx64.efi" > $BOOTFS_MNT/startup.nsh + + +# Call cleanup to unmount devices and images and remove the TMPDIR +cleanup + +echo "" +if [ $WARNINGS -ne 0 ] && [ $ERRORS -eq 0 ]; then + echo "${YELLOW}Installation completed with warnings${CLEAR}" + echo "${YELLOW}Warnings: $WARNINGS${CLEAR}" +elif [ $ERRORS -ne 0 ]; then + echo "${RED}Installation encountered errors${CLEAR}" + echo "${RED}Errors: $ERRORS${CLEAR}" + echo "${YELLOW}Warnings: $WARNINGS${CLEAR}" +else + success "Installation completed successfully" +fi +echo "" diff --git a/scripts/contrib/python/generate-manifest-2.7.py b/scripts/contrib/python/generate-manifest-2.7.py new file mode 100755 index 0000000..d93c943 --- /dev/null +++ b/scripts/contrib/python/generate-manifest-2.7.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python + +# generate Python Manifest for the OpenEmbedded build system +# (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> +# (C) 2007 Jeremy Laine +# licensed under MIT, see COPYING.MIT +# +# June 22, 2011 -- Mark Hatle <mark.hatle@windriver.com> +# * Updated to no longer generate special -dbg package, instead use the +# single system -dbg +# * Update version with ".1" to indicate this change + +import os +import sys +import time + +VERSION = "2.7.2" + +__author__ = "Michael 'Mickey' Lauer <mlauer@vanille-media.de>" +__version__ = "20110222.2" + +class MakefileMaker: + + def __init__( self, outfile ): + """initialize""" + self.packages = {} + self.targetPrefix = "${libdir}/python%s/" % VERSION[:3] + self.output = outfile + self.out( """ +# WARNING: This file is AUTO GENERATED: Manual edits will be lost next time I regenerate the file. +# Generator: '%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> +# Visit the Python for Embedded Systems Site => http://www.Vanille.de/projects/python.spy +""" % ( sys.argv[0], __version__ ) ) + + # + # helper functions + # + + def out( self, data ): + """print a line to the output file""" + self.output.write( "%s\n" % data ) + + def setPrefix( self, targetPrefix ): + """set a file prefix for addPackage files""" + self.targetPrefix = targetPrefix + + def doProlog( self ): + self.out( """ """ ) + self.out( "" ) + + def addPackage( self, name, description, dependencies, filenames ): + """add a package to the Makefile""" + if type( filenames ) == type( "" ): + filenames = filenames.split() + fullFilenames = [] + for filename in filenames: + if filename[0] != "$": + fullFilenames.append( "%s%s" % ( self.targetPrefix, filename ) ) + else: + fullFilenames.append( filename ) + self.packages[name] = description, dependencies, fullFilenames + + def doBody( self ): + """generate body of Makefile""" + + global VERSION + + # + # generate provides line + # + + provideLine = 'PROVIDES+="' + for name in sorted(self.packages): + provideLine += "%s " % name + provideLine += '"' + + self.out( provideLine ) + self.out( "" ) + + # + # generate package line + # + + packageLine = 'PACKAGES="${PN}-dbg ' + for name in sorted(self.packages): + if name.startswith("${PN}-distutils"): + if name == "${PN}-distutils": + packageLine += "%s-staticdev %s " % (name, name) + elif name != '${PN}-dbg': + packageLine += "%s " % name + packageLine += '${PN}-modules"' + + self.out( packageLine ) + self.out( "" ) + + # + # generate package variables + # + + for name, data in sorted(self.packages.iteritems()): + desc, deps, files = data + + # + # write out the description, revision and dependencies + # + self.out( 'SUMMARY_%s="%s"' % ( name, desc ) ) + self.out( 'RDEPENDS_%s="%s"' % ( name, deps ) ) + + line = 'FILES_%s="' % name + + # + # check which directories to make in the temporary directory + # + + dirset = {} # if python had a set-datatype this would be sufficient. for now, we're using a dict instead. + for target in files: + dirset[os.path.dirname( target )] = True + + # + # generate which files to copy for the target (-dfR because whole directories are also allowed) + # + + for target in files: + line += "%s " % target + + line += '"' + self.out( line ) + self.out( "" ) + + self.out( 'SUMMARY_${PN}-modules="All Python modules"' ) + line = 'RDEPENDS_${PN}-modules="' + + for name, data in sorted(self.packages.iteritems()): + if name not in ['${PN}-dev', '${PN}-distutils-staticdev']: + line += "%s " % name + + self.out( "%s \"" % line ) + self.out( 'ALLOW_EMPTY_${PN}-modules = "1"' ) + + def doEpilog( self ): + self.out( """""" ) + self.out( "" ) + + def make( self ): + self.doProlog() + self.doBody() + self.doEpilog() + +if __name__ == "__main__": + + if len( sys.argv ) > 1: + try: + os.unlink(sys.argv[1]) + except Exception: + sys.exc_clear() + outfile = file( sys.argv[1], "w" ) + else: + outfile = sys.stdout + + m = MakefileMaker( outfile ) + + # Add packages here. Only specify dlopen-style library dependencies here, no ldd-style dependencies! + # Parameters: revision, name, description, dependencies, filenames + # + + m.addPackage( "${PN}-core", "Python interpreter and core modules", "${PN}-lang ${PN}-re", + "__future__.* _abcoll.* abc.* ast.* copy.* copy_reg.* ConfigParser.* " + + "genericpath.* getopt.* linecache.* new.* " + + "os.* posixpath.* struct.* " + + "warnings.* site.* stat.* " + + "UserDict.* UserList.* UserString.* " + + "lib-dynload/binascii.so lib-dynload/_struct.so lib-dynload/time.so " + + "lib-dynload/xreadlines.so types.* platform.* ${bindir}/python* " + + "_weakrefset.* sysconfig.* _sysconfigdata.* config/Makefile " + + "${includedir}/python${PYTHON_MAJMIN}/pyconfig*.h " + + "${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py ") + + m.addPackage( "${PN}-dev", "Python development package", "${PN}-core", + "${includedir} " + + "${libdir}/lib*${SOLIBSDEV} " + + "${libdir}/*.la " + + "${libdir}/*.a " + + "${libdir}/*.o " + + "${libdir}/pkgconfig " + + "${base_libdir}/*.a " + + "${base_libdir}/*.o " + + "${datadir}/aclocal " + + "${datadir}/pkgconfig " ) + + m.addPackage( "${PN}-2to3", "Python automated Python 2 to 3 code translator", "${PN}-core", + "${bindir}/2to3 lib2to3" ) # package + + m.addPackage( "${PN}-idle", "Python Integrated Development Environment", "${PN}-core ${PN}-tkinter", + "${bindir}/idle idlelib" ) # package + + m.addPackage( "${PN}-pydoc", "Python interactive help support", "${PN}-core ${PN}-lang ${PN}-stringold ${PN}-re", + "${bindir}/pydoc pydoc.* pydoc_data" ) + + m.addPackage( "${PN}-smtpd", "Python Simple Mail Transport Daemon", "${PN}-core ${PN}-netserver ${PN}-email ${PN}-mime", + "${bindir}/smtpd.* smtpd.*" ) + + m.addPackage( "${PN}-audio", "Python Audio Handling", "${PN}-core", + "wave.* chunk.* sndhdr.* lib-dynload/ossaudiodev.so lib-dynload/audioop.so audiodev.* sunaudio.* sunau.* toaiff.*" ) + + m.addPackage( "${PN}-bsddb", "Python bindings for the Berkeley Database", "${PN}-core", + "bsddb lib-dynload/_bsddb.so" ) # package + + m.addPackage( "${PN}-codecs", "Python codecs, encodings & i18n support", "${PN}-core ${PN}-lang", + "codecs.* encodings gettext.* locale.* lib-dynload/_locale.so lib-dynload/_codecs* lib-dynload/_multibytecodec.so lib-dynload/unicodedata.so stringprep.* xdrlib.*" ) + + m.addPackage( "${PN}-compile", "Python bytecode compilation support", "${PN}-core", + "py_compile.* compileall.*" ) + + m.addPackage( "${PN}-compiler", "Python compiler support", "${PN}-core", + "compiler" ) # package + + m.addPackage( "${PN}-compression", "Python high-level compression support", "${PN}-core ${PN}-zlib", + "gzip.* zipfile.* tarfile.* lib-dynload/bz2.so" ) + + m.addPackage( "${PN}-crypt", "Python basic cryptographic and hashing support", "${PN}-core", + "hashlib.* md5.* sha.* lib-dynload/crypt.so lib-dynload/_hashlib.so lib-dynload/_sha256.so lib-dynload/_sha512.so" ) + + m.addPackage( "${PN}-textutils", "Python option parsing, text wrapping and CSV support", "${PN}-core ${PN}-io ${PN}-re ${PN}-stringold", + "lib-dynload/_csv.so csv.* optparse.* textwrap.*" ) + + m.addPackage( "${PN}-curses", "Python curses support", "${PN}-core", + "curses lib-dynload/_curses.so lib-dynload/_curses_panel.so" ) # directory + low level module + + m.addPackage( "${PN}-ctypes", "Python C types support", "${PN}-core", + "ctypes lib-dynload/_ctypes.so lib-dynload/_ctypes_test.so" ) # directory + low level module + + m.addPackage( "${PN}-datetime", "Python calendar and time support", "${PN}-core ${PN}-codecs", + "_strptime.* calendar.* lib-dynload/datetime.so" ) + + m.addPackage( "${PN}-db", "Python file-based database support", "${PN}-core", + "anydbm.* dumbdbm.* whichdb.* " ) + + m.addPackage( "${PN}-debugger", "Python debugger", "${PN}-core ${PN}-io ${PN}-lang ${PN}-re ${PN}-stringold ${PN}-shell ${PN}-pprint", + "bdb.* pdb.*" ) + + m.addPackage( "${PN}-difflib", "Python helpers for computing deltas between objects", "${PN}-lang ${PN}-re", + "difflib.*" ) + + m.addPackage( "${PN}-distutils-staticdev", "Python distribution utilities (static libraries)", "${PN}-distutils", + "config/lib*.a" ) # package + + m.addPackage( "${PN}-distutils", "Python Distribution Utilities", "${PN}-core ${PN}-email", + "config distutils" ) # package + + m.addPackage( "${PN}-doctest", "Python framework for running examples in docstrings", "${PN}-core ${PN}-lang ${PN}-io ${PN}-re ${PN}-unittest ${PN}-debugger ${PN}-difflib", + "doctest.*" ) + + m.addPackage( "${PN}-email", "Python email support", "${PN}-core ${PN}-io ${PN}-re ${PN}-mime ${PN}-audio ${PN}-image ${PN}-netclient", + "imaplib.* email" ) # package + + m.addPackage( "${PN}-fcntl", "Python's fcntl interface", "${PN}-core", + "lib-dynload/fcntl.so" ) + + m.addPackage( "${PN}-hotshot", "Python hotshot performance profiler", "${PN}-core", + "hotshot lib-dynload/_hotshot.so" ) + + m.addPackage( "${PN}-html", "Python HTML processing support", "${PN}-core", + "formatter.* htmlentitydefs.* htmllib.* markupbase.* sgmllib.* HTMLParser.* " ) + + m.addPackage( "${PN}-importlib", "Python import implementation library", "${PN}-core", + "importlib" ) + + m.addPackage( "${PN}-gdbm", "Python GNU database support", "${PN}-core", + "lib-dynload/gdbm.so" ) + + m.addPackage( "${PN}-image", "Python graphical image handling", "${PN}-core", + "colorsys.* imghdr.* lib-dynload/imageop.so lib-dynload/rgbimg.so" ) + + m.addPackage( "${PN}-io", "Python low-level I/O", "${PN}-core ${PN}-math ${PN}-textutils ${PN}-netclient ${PN}-contextlib", + "lib-dynload/_socket.so lib-dynload/_io.so lib-dynload/_ssl.so lib-dynload/select.so lib-dynload/termios.so lib-dynload/cStringIO.so " + + "pipes.* socket.* ssl.* tempfile.* StringIO.* io.* _pyio.*" ) + + m.addPackage( "${PN}-json", "Python JSON support", "${PN}-core ${PN}-math ${PN}-re ${PN}-codecs", + "json lib-dynload/_json.so" ) # package + + m.addPackage( "${PN}-lang", "Python low-level language support", "${PN}-core", + "lib-dynload/_bisect.so lib-dynload/_collections.so lib-dynload/_heapq.so lib-dynload/_weakref.so lib-dynload/_functools.so " + + "lib-dynload/array.so lib-dynload/itertools.so lib-dynload/operator.so lib-dynload/parser.so " + + "atexit.* bisect.* code.* codeop.* collections.* dis.* functools.* heapq.* inspect.* keyword.* opcode.* symbol.* repr.* token.* " + + "tokenize.* traceback.* weakref.*" ) + + m.addPackage( "${PN}-logging", "Python logging support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-stringold", + "logging" ) # package + + m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", + "mailbox.*" ) + + m.addPackage( "${PN}-math", "Python math support", "${PN}-core ${PN}-crypt", + "lib-dynload/cmath.so lib-dynload/math.so lib-dynload/_random.so random.* sets.*" ) + + m.addPackage( "${PN}-mime", "Python MIME handling APIs", "${PN}-core ${PN}-io", + "mimetools.* uu.* quopri.* rfc822.* MimeWriter.*" ) + + m.addPackage( "${PN}-mmap", "Python memory-mapped file support", "${PN}-core ${PN}-io", + "lib-dynload/mmap.so " ) + + m.addPackage( "${PN}-multiprocessing", "Python multiprocessing support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-threading ${PN}-ctypes ${PN}-mmap", + "lib-dynload/_multiprocessing.so multiprocessing" ) # package + + m.addPackage( "${PN}-netclient", "Python Internet Protocol clients", "${PN}-core ${PN}-crypt ${PN}-datetime ${PN}-io ${PN}-lang ${PN}-logging ${PN}-mime", + "*Cookie*.* " + + "base64.* cookielib.* ftplib.* gopherlib.* hmac.* httplib.* mimetypes.* nntplib.* poplib.* smtplib.* telnetlib.* urllib.* urllib2.* urlparse.* uuid.* rfc822.* mimetools.*" ) + + m.addPackage( "${PN}-netserver", "Python Internet Protocol servers", "${PN}-core ${PN}-netclient ${PN}-shell ${PN}-threading", + "cgi.* *HTTPServer.* SocketServer.*" ) + + m.addPackage( "${PN}-numbers", "Python number APIs", "${PN}-core ${PN}-lang ${PN}-re", + "decimal.* fractions.* numbers.*" ) + + m.addPackage( "${PN}-pickle", "Python serialisation/persistence support", "${PN}-core ${PN}-codecs ${PN}-io ${PN}-re", + "pickle.* shelve.* lib-dynload/cPickle.so pickletools.*" ) + + m.addPackage( "${PN}-pkgutil", "Python package extension utility support", "${PN}-core", + "pkgutil.*") + + m.addPackage( "${PN}-plistlib", "Generate and parse Mac OS X .plist files", "${PN}-core ${PN}-datetime ${PN}-io", + "plistlib.*") + + m.addPackage( "${PN}-pprint", "Python pretty-print support", "${PN}-core ${PN}-io", + "pprint.*" ) + + m.addPackage( "${PN}-profile", "Python basic performance profiling support", "${PN}-core ${PN}-textutils", + "profile.* pstats.* cProfile.* lib-dynload/_lsprof.so" ) + + m.addPackage( "${PN}-re", "Python Regular Expression APIs", "${PN}-core", + "re.* sre.* sre_compile.* sre_constants* sre_parse.*" ) # _sre is builtin + + m.addPackage( "${PN}-readline", "Python readline support", "${PN}-core", + "lib-dynload/readline.so rlcompleter.*" ) + + m.addPackage( "${PN}-resource", "Python resource control interface", "${PN}-core", + "lib-dynload/resource.so" ) + + m.addPackage( "${PN}-shell", "Python shell-like functionality", "${PN}-core ${PN}-re", + "cmd.* commands.* dircache.* fnmatch.* glob.* popen2.* shlex.* shutil.*" ) + + m.addPackage( "${PN}-robotparser", "Python robots.txt parser", "${PN}-core ${PN}-netclient", + "robotparser.*") + + m.addPackage( "${PN}-subprocess", "Python subprocess support", "${PN}-core ${PN}-io ${PN}-re ${PN}-fcntl ${PN}-pickle", + "subprocess.*" ) + + m.addPackage( "${PN}-sqlite3", "Python Sqlite3 database support", "${PN}-core ${PN}-datetime ${PN}-lang ${PN}-crypt ${PN}-io ${PN}-threading ${PN}-zlib", + "lib-dynload/_sqlite3.so sqlite3/dbapi2.* sqlite3/__init__.* sqlite3/dump.*" ) + + m.addPackage( "${PN}-sqlite3-tests", "Python Sqlite3 database support tests", "${PN}-core ${PN}-sqlite3", + "sqlite3/test" ) + + m.addPackage( "${PN}-stringold", "Python string APIs [deprecated]", "${PN}-core ${PN}-re", + "lib-dynload/strop.so string.* stringold.*" ) + + m.addPackage( "${PN}-syslog", "Python syslog interface", "${PN}-core", + "lib-dynload/syslog.so" ) + + m.addPackage( "${PN}-terminal", "Python terminal controlling support", "${PN}-core ${PN}-io", + "pty.* tty.*" ) + + m.addPackage( "${PN}-tests", "Python tests", "${PN}-core", + "test" ) # package + + m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang", + "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* Queue.*" ) + + m.addPackage( "${PN}-tkinter", "Python Tcl/Tk bindings", "${PN}-core", + "lib-dynload/_tkinter.so lib-tk" ) # package + + m.addPackage( "${PN}-unittest", "Python unit testing framework", "${PN}-core ${PN}-stringold ${PN}-lang ${PN}-io ${PN}-difflib ${PN}-pprint ${PN}-shell", + "unittest/" ) + + m.addPackage( "${PN}-unixadmin", "Python Unix administration support", "${PN}-core", + "lib-dynload/nis.so lib-dynload/grp.so lib-dynload/pwd.so getpass.*" ) + + m.addPackage( "${PN}-xml", "Python basic XML support", "${PN}-core ${PN}-re", + "lib-dynload/_elementtree.so lib-dynload/pyexpat.so xml xmllib.*" ) # package + + m.addPackage( "${PN}-xmlrpc", "Python XML-RPC support", "${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang", + "xmlrpclib.* SimpleXMLRPCServer.* DocXMLRPCServer.*" ) + + m.addPackage( "${PN}-zlib", "Python zlib compression support", "${PN}-core", + "lib-dynload/zlib.so" ) + + m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", + "mailbox.*" ) + + m.addPackage( "${PN}-argparse", "Python command line argument parser", "${PN}-core ${PN}-codecs ${PN}-textutils", + "argparse.*" ) + + m.addPackage( "${PN}-contextlib", "Python utilities for with-statement" + + "contexts.", "${PN}-core", + "${libdir}/python${PYTHON_MAJMIN}/contextlib.*" ) + + m.make() diff --git a/scripts/contrib/python/generate-manifest-3.5.py b/scripts/contrib/python/generate-manifest-3.5.py new file mode 100755 index 0000000..eac493a --- /dev/null +++ b/scripts/contrib/python/generate-manifest-3.5.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python + +# generate Python Manifest for the OpenEmbedded build system +# (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> +# (C) 2007 Jeremy Laine +# licensed under MIT, see COPYING.MIT +# +# June 22, 2011 -- Mark Hatle <mark.hatle@windriver.com> +# * Updated to no longer generate special -dbg package, instead use the +# single system -dbg +# * Update version with ".1" to indicate this change +# +# 2014 Khem Raj <raj.khem@gmail.com> +# Added python3 support +# +import os +import sys +import time + +VERSION = "3.5.0" + +__author__ = "Michael 'Mickey' Lauer <mlauer@vanille-media.de>" +__version__ = "20140131" + +class MakefileMaker: + + def __init__( self, outfile ): + """initialize""" + self.packages = {} + self.targetPrefix = "${libdir}/python%s/" % VERSION[:3] + self.output = outfile + self.out( """ +# WARNING: This file is AUTO GENERATED: Manual edits will be lost next time I regenerate the file. +# Generator: '%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> +# Visit the Python for Embedded Systems Site => http://www.Vanille.de/projects/python.spy +""" % ( sys.argv[0], __version__ ) ) + + # + # helper functions + # + + def out( self, data ): + """print a line to the output file""" + self.output.write( "%s\n" % data ) + + def setPrefix( self, targetPrefix ): + """set a file prefix for addPackage files""" + self.targetPrefix = targetPrefix + + def doProlog( self ): + self.out( """ """ ) + self.out( "" ) + + def addPackage( self, name, description, dependencies, filenames ): + """add a package to the Makefile""" + if type( filenames ) == type( "" ): + filenames = filenames.split() + fullFilenames = [] + for filename in filenames: + if filename[0] != "$": + fullFilenames.append( "%s%s" % ( self.targetPrefix, filename ) ) + else: + fullFilenames.append( filename ) + self.packages[name] = description, dependencies, fullFilenames + + def doBody( self ): + """generate body of Makefile""" + + global VERSION + + # + # generate provides line + # + + provideLine = 'PROVIDES+="' + for name in sorted(self.packages): + provideLine += "%s " % name + provideLine += '"' + + self.out( provideLine ) + self.out( "" ) + + # + # generate package line + # + + packageLine = 'PACKAGES="${PN}-dbg ' + for name in sorted(self.packages): + if name.startswith("${PN}-distutils"): + if name == "${PN}-distutils": + packageLine += "%s-staticdev %s " % (name, name) + elif name != '${PN}-dbg': + packageLine += "%s " % name + packageLine += '${PN}-modules"' + + self.out( packageLine ) + self.out( "" ) + + # + # generate package variables + # + + for name, data in sorted(self.packages.iteritems()): + desc, deps, files = data + + # + # write out the description, revision and dependencies + # + self.out( 'SUMMARY_%s="%s"' % ( name, desc ) ) + self.out( 'RDEPENDS_%s="%s"' % ( name, deps ) ) + + line = 'FILES_%s="' % name + + # + # check which directories to make in the temporary directory + # + + dirset = {} # if python had a set-datatype this would be sufficient. for now, we're using a dict instead. + for target in files: + dirset[os.path.dirname( target )] = True + + # + # generate which files to copy for the target (-dfR because whole directories are also allowed) + # + + for target in files: + line += "%s " % target + + line += '"' + self.out( line ) + self.out( "" ) + + self.out( 'SUMMARY_${PN}-modules="All Python modules"' ) + line = 'RDEPENDS_${PN}-modules="' + + for name, data in sorted(self.packages.iteritems()): + if name not in ['${PN}-dev', '${PN}-distutils-staticdev']: + line += "%s " % name + + self.out( "%s \"" % line ) + self.out( 'ALLOW_EMPTY_${PN}-modules = "1"' ) + + def doEpilog( self ): + self.out( """""" ) + self.out( "" ) + + def make( self ): + self.doProlog() + self.doBody() + self.doEpilog() + +if __name__ == "__main__": + + if len( sys.argv ) > 1: + try: + os.unlink(sys.argv[1]) + except Exception: + sys.exc_clear() + outfile = file( sys.argv[1], "w" ) + else: + outfile = sys.stdout + + m = MakefileMaker( outfile ) + + # Add packages here. Only specify dlopen-style library dependencies here, no ldd-style dependencies! + # Parameters: revision, name, description, dependencies, filenames + # + + m.addPackage( "${PN}-core", "Python interpreter and core modules", "${PN}-lang ${PN}-re ${PN}-reprlib ${PN}-codecs ${PN}-io ${PN}-math", + "__future__.* _abcoll.* abc.* ast.* copy.* copyreg.* ConfigParser.* " + + "genericpath.* getopt.* linecache.* new.* " + + "os.* posixpath.* struct.* " + + "warnings.* site.* stat.* " + + "UserDict.* UserList.* UserString.* " + + "lib-dynload/binascii.*.so lib-dynload/_struct.*.so lib-dynload/time.*.so " + + "lib-dynload/xreadlines.*.so types.* platform.* ${bindir}/python* " + + "_weakrefset.* sysconfig.* _sysconfigdata.* config/Makefile " + + "${includedir}/python${PYTHON_BINABI}/pyconfig*.h " + + "${libdir}/python${PYTHON_MAJMIN}/collections " + + "${libdir}/python${PYTHON_MAJMIN}/_collections_abc.* " + + "${libdir}/python${PYTHON_MAJMIN}/_sitebuiltins.* " + + "${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py ") + + m.addPackage( "${PN}-dev", "Python development package", "${PN}-core", + "${includedir} " + + "${libdir}/lib*${SOLIBSDEV} " + + "${libdir}/*.la " + + "${libdir}/*.a " + + "${libdir}/*.o " + + "${libdir}/pkgconfig " + + "${base_libdir}/*.a " + + "${base_libdir}/*.o " + + "${datadir}/aclocal " + + "${datadir}/pkgconfig " ) + + m.addPackage( "${PN}-2to3", "Python automated Python 2 to 3 code translator", "${PN}-core", + "lib2to3" ) # package + + m.addPackage( "${PN}-idle", "Python Integrated Development Environment", "${PN}-core ${PN}-tkinter", + "${bindir}/idle idlelib" ) # package + + m.addPackage( "${PN}-pydoc", "Python interactive help support", "${PN}-core ${PN}-lang ${PN}-stringold ${PN}-re", + "${bindir}/pydoc pydoc.* pydoc_data" ) + + m.addPackage( "${PN}-smtpd", "Python Simple Mail Transport Daemon", "${PN}-core ${PN}-netserver ${PN}-email ${PN}-mime", + "${bindir}/smtpd.* smtpd.*" ) + + m.addPackage( "${PN}-audio", "Python Audio Handling", "${PN}-core", + "wave.* chunk.* sndhdr.* lib-dynload/ossaudiodev.*.so lib-dynload/audioop.*.so audiodev.* sunaudio.* sunau.* toaiff.*" ) + + m.addPackage( "${PN}-argparse", "Python command line argument parser", "${PN}-core ${PN}-codecs ${PN}-textutils", + "argparse.*" ) + + m.addPackage( "${PN}-asyncio", "Python Asynchronous I/O, event loop, coroutines and tasks", "${PN}-core", + "asyncio" ) + + m.addPackage( "${PN}-codecs", "Python codecs, encodings & i18n support", "${PN}-core ${PN}-lang", + "codecs.* encodings gettext.* locale.* lib-dynload/_locale.*.so lib-dynload/_codecs* lib-dynload/_multibytecodec.*.so lib-dynload/unicodedata.*.so stringprep.* xdrlib.*" ) + + m.addPackage( "${PN}-compile", "Python bytecode compilation support", "${PN}-core", + "py_compile.* compileall.*" ) + + m.addPackage( "${PN}-compression", "Python high-level compression support", "${PN}-core ${PN}-codecs ${PN}-importlib ${PN}-threading ${PN}-shell", + "gzip.* zipfile.* tarfile.* lib-dynload/bz2.*.so" ) + + m.addPackage( "${PN}-crypt", "Python basic cryptographic and hashing support", "${PN}-core", + "hashlib.* md5.* sha.* lib-dynload/crypt.*.so lib-dynload/_hashlib.*.so lib-dynload/_sha256.*.so lib-dynload/_sha512.*.so" ) + + m.addPackage( "${PN}-textutils", "Python option parsing, text wrapping and CSV support", "${PN}-core ${PN}-io ${PN}-re ${PN}-stringold", + "lib-dynload/_csv.*.so csv.* optparse.* textwrap.*" ) + + m.addPackage( "${PN}-curses", "Python curses support", "${PN}-core", + "curses lib-dynload/_curses.*.so lib-dynload/_curses_panel.*.so" ) # directory + low level module + + m.addPackage( "${PN}-ctypes", "Python C types support", "${PN}-core", + "ctypes lib-dynload/_ctypes.*.so lib-dynload/_ctypes_test.*.so" ) # directory + low level module + + m.addPackage( "${PN}-datetime", "Python calendar and time support", "${PN}-core ${PN}-codecs", + "_strptime.* calendar.* lib-dynload/datetime.*.so" ) + + m.addPackage( "${PN}-db", "Python file-based database support", "${PN}-core", + "anydbm.* dumbdbm.* whichdb.* dbm lib-dynload/_dbm.*.so" ) + + m.addPackage( "${PN}-debugger", "Python debugger", "${PN}-core ${PN}-io ${PN}-lang ${PN}-re ${PN}-stringold ${PN}-shell ${PN}-pprint ${PN}-importlib ${PN}-pkgutil", + "bdb.* pdb.*" ) + + m.addPackage( "${PN}-difflib", "Python helpers for computing deltas between objects", "${PN}-lang ${PN}-re", + "difflib.*" ) + + m.addPackage( "${PN}-distutils-staticdev", "Python distribution utilities (static libraries)", "${PN}-distutils", + "config/lib*.a" ) # package + + m.addPackage( "${PN}-distutils", "Python Distribution Utilities", "${PN}-core ${PN}-email", + "config distutils" ) # package + + m.addPackage( "${PN}-doctest", "Python framework for running examples in docstrings", "${PN}-core ${PN}-lang ${PN}-io ${PN}-re ${PN}-unittest ${PN}-debugger ${PN}-difflib", + "doctest.*" ) + + m.addPackage( "${PN}-email", "Python email support", "${PN}-core ${PN}-io ${PN}-re ${PN}-mime ${PN}-audio ${PN}-image ${PN}-netclient", + "imaplib.* email" ) # package + + m.addPackage( "${PN}-enum", "Python support for enumerations", "${PN}-core", + "enum.*" ) + + m.addPackage( "${PN}-fcntl", "Python's fcntl interface", "${PN}-core", + "lib-dynload/fcntl.*.so" ) + + m.addPackage( "${PN}-html", "Python HTML processing support", "${PN}-core", + "formatter.* htmlentitydefs.* htmllib.* markupbase.* sgmllib.* HTMLParser.* " ) + + m.addPackage( "${PN}-importlib", "Python import implementation library", "${PN}-core ${PN}-lang", + "importlib" ) + + m.addPackage( "${PN}-gdbm", "Python GNU database support", "${PN}-core", + "lib-dynload/_gdbm.*.so" ) + + m.addPackage( "${PN}-image", "Python graphical image handling", "${PN}-core", + "colorsys.* imghdr.* lib-dynload/imageop.*.so lib-dynload/rgbimg.*.so" ) + + m.addPackage( "${PN}-io", "Python low-level I/O", "${PN}-core ${PN}-math", + "lib-dynload/_socket.*.so lib-dynload/_io.*.so lib-dynload/_ssl.*.so lib-dynload/select.*.so lib-dynload/termios.*.so lib-dynload/cStringIO.*.so " + + "pipes.* socket.* ssl.* tempfile.* StringIO.* io.* _pyio.*" ) + + m.addPackage( "${PN}-json", "Python JSON support", "${PN}-core ${PN}-math ${PN}-re", + "json lib-dynload/_json.*.so" ) # package + + m.addPackage( "${PN}-lang", "Python low-level language support", "${PN}-core", + "lib-dynload/_bisect.*.so lib-dynload/_collections.*.so lib-dynload/_heapq.*.so lib-dynload/_weakref.*.so lib-dynload/_functools.*.so " + + "lib-dynload/array.*.so lib-dynload/itertools.*.so lib-dynload/operator.*.so lib-dynload/parser.*.so " + + "atexit.* bisect.* code.* codeop.* collections.* _collections_abc.* contextlib.* dis.* functools.* heapq.* inspect.* keyword.* opcode.* operator.* symbol.* repr.* token.* " + + "tokenize.* traceback.* weakref.*" ) + + m.addPackage( "${PN}-logging", "Python logging support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-stringold", + "logging" ) # package + + m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", + "mailbox.*" ) + + m.addPackage( "${PN}-math", "Python math support", "${PN}-core", + "lib-dynload/cmath.*.so lib-dynload/math.*.so lib-dynload/_random.*.so random.* sets.*" ) + + m.addPackage( "${PN}-mime", "Python MIME handling APIs", "${PN}-core ${PN}-io", + "mimetools.* uu.* quopri.* rfc822.* MimeWriter.*" ) + + m.addPackage( "${PN}-mmap", "Python memory-mapped file support", "${PN}-core ${PN}-io", + "lib-dynload/mmap.*.so " ) + + m.addPackage( "${PN}-multiprocessing", "Python multiprocessing support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-threading ${PN}-ctypes ${PN}-mmap", + "lib-dynload/_multiprocessing.*.so multiprocessing" ) # package + + m.addPackage( "${PN}-netclient", "Python Internet Protocol clients", "${PN}-core ${PN}-crypt ${PN}-datetime ${PN}-io ${PN}-lang ${PN}-logging ${PN}-mime", + "*Cookie*.* " + + "base64.* cookielib.* ftplib.* gopherlib.* hmac.* httplib.* mimetypes.* nntplib.* poplib.* smtplib.* telnetlib.* urllib uuid.* rfc822.* mimetools.*" ) + + m.addPackage( "${PN}-netserver", "Python Internet Protocol servers", "${PN}-core ${PN}-netclient ${PN}-shell ${PN}-threading", + "cgi.* *HTTPServer.* SocketServer.*" ) + + m.addPackage( "${PN}-numbers", "Python number APIs", "${PN}-core ${PN}-lang ${PN}-re", + "decimal.* fractions.* numbers.*" ) + + m.addPackage( "${PN}-pickle", "Python serialisation/persistence support", "${PN}-core ${PN}-codecs ${PN}-io ${PN}-re", + "pickle.* shelve.* lib-dynload/cPickle.*.so pickletools.*" ) + + m.addPackage( "${PN}-pkgutil", "Python package extension utility support", "${PN}-core", + "pkgutil.*") + + m.addPackage( "${PN}-pprint", "Python pretty-print support", "${PN}-core ${PN}-io", + "pprint.*" ) + + m.addPackage( "${PN}-profile", "Python basic performance profiling support", "${PN}-core ${PN}-textutils", + "profile.* pstats.* cProfile.* lib-dynload/_lsprof.*.so" ) + + m.addPackage( "${PN}-re", "Python Regular Expression APIs", "${PN}-core", + "re.* sre.* sre_compile.* sre_constants* sre_parse.*" ) # _sre is builtin + + m.addPackage( "${PN}-readline", "Python readline support", "${PN}-core", + "lib-dynload/readline.*.so rlcompleter.*" ) + + m.addPackage( "${PN}-reprlib", "Python alternate repr() implementation", "${PN}-core", + "reprlib.py" ) + + m.addPackage( "${PN}-resource", "Python resource control interface", "${PN}-core", + "lib-dynload/resource.*.so" ) + + m.addPackage( "${PN}-selectors", "Python High-level I/O multiplexing", "${PN}-core", + "selectors.*" ) + + m.addPackage( "${PN}-shell", "Python shell-like functionality", "${PN}-core ${PN}-re", + "cmd.* commands.* dircache.* fnmatch.* glob.* popen2.* shlex.* shutil.*" ) + + m.addPackage( "${PN}-subprocess", "Python subprocess support", "${PN}-core ${PN}-io ${PN}-re ${PN}-fcntl ${PN}-pickle ${PN}-signal ${PN}-selectors", + "subprocess.*" ) + + m.addPackage( "${PN}-signal", "Python set handlers for asynchronous events support", "${PN}-core", + "signal.*" ) + + m.addPackage( "${PN}-sqlite3", "Python Sqlite3 database support", "${PN}-core ${PN}-datetime ${PN}-lang ${PN}-crypt ${PN}-io ${PN}-threading", + "lib-dynload/_sqlite3.*.so sqlite3/dbapi2.* sqlite3/__init__.* sqlite3/dump.*" ) + + m.addPackage( "${PN}-sqlite3-tests", "Python Sqlite3 database support tests", "${PN}-core ${PN}-sqlite3", + "sqlite3/test" ) + + m.addPackage( "${PN}-stringold", "Python string APIs [deprecated]", "${PN}-core ${PN}-re", + "lib-dynload/strop.*.so string.* stringold.*" ) + + m.addPackage( "${PN}-syslog", "Python syslog interface", "${PN}-core", + "lib-dynload/syslog.*.so" ) + + m.addPackage( "${PN}-terminal", "Python terminal controlling support", "${PN}-core ${PN}-io", + "pty.* tty.*" ) + + m.addPackage( "${PN}-tests", "Python tests", "${PN}-core", + "test" ) # package + + m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang", + "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* queue.*" ) + + m.addPackage( "${PN}-tkinter", "Python Tcl/Tk bindings", "${PN}-core", + "lib-dynload/_tkinter.*.so lib-tk tkinter" ) # package + + m.addPackage( "${PN}-unittest", "Python unit testing framework", "${PN}-core ${PN}-stringold ${PN}-lang ${PN}-io ${PN}-difflib ${PN}-pprint ${PN}-shell", + "unittest/" ) + + m.addPackage( "${PN}-unixadmin", "Python Unix administration support", "${PN}-core", + "lib-dynload/nis.*.so lib-dynload/grp.*.so lib-dynload/pwd.*.so getpass.*" ) + + m.addPackage( "${PN}-xml", "Python basic XML support", "${PN}-core ${PN}-re", + "lib-dynload/_elementtree.*.so lib-dynload/pyexpat.*.so xml xmllib.*" ) # package + + m.addPackage( "${PN}-xmlrpc", "Python XML-RPC support", "${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang", + "xmlrpclib.* SimpleXMLRPCServer.* DocXMLRPCServer.* xmlrpc" ) + + m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", + "mailbox.*" ) + + m.make() diff --git a/scripts/contrib/serdevtry b/scripts/contrib/serdevtry new file mode 100755 index 0000000..74bd7b7 --- /dev/null +++ b/scripts/contrib/serdevtry @@ -0,0 +1,60 @@ +#!/bin/sh + +# Copyright (C) 2014 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +if [ "$1" = "" -o "$1" = "--help" ] ; then + echo "Usage: $0 <serial terminal command>" + echo + echo "Simple script to handle maintaining a terminal for serial devices that" + echo "disappear when a device is powered down or reset, such as the USB" + echo "serial console on the original BeagleBone (white version)." + echo + echo "e.g. $0 picocom -b 115200 /dev/ttyUSB0" + echo + exit +fi + +args="$@" +DEVICE="" +while [ "$1" != "" ]; do + case "$1" in + /dev/*) + DEVICE=$1 + break;; + esac + shift +done + +if [ "$DEVICE" != "" ] ; then + while true; do + if [ ! -e $DEVICE ] ; then + echo "serdevtry: waiting for $DEVICE to exist..." + while [ ! -e $DEVICE ]; do + sleep 0.1 + done + fi + if [ ! -w $DEVICE ] ; then + # Sometimes (presumably because of a race with udev) we get to + # the device before its permissions have been set up + RETRYNUM=0 + while [ ! -w $DEVICE ]; do + if [ "$RETRYNUM" = "2" ] ; then + echo "Device $DEVICE exists but is not writable!" + exit 1 + fi + RETRYNUM=$((RETRYNUM+1)) + sleep 0.1 + done + fi + $args + if [ -e $DEVICE ] ; then + break + fi + done +else + echo "Unable to determine device node from command: $args" + exit 1 +fi + diff --git a/scripts/contrib/test_build_time.sh b/scripts/contrib/test_build_time.sh new file mode 100755 index 0000000..9e5725a --- /dev/null +++ b/scripts/contrib/test_build_time.sh @@ -0,0 +1,237 @@ +#!/bin/bash + +# Build performance regression test script +# +# Copyright 2011 Intel Corporation +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# +# DESCRIPTION +# This script is intended to be used in conjunction with "git bisect run" +# in order to find regressions in build time, however it can also be used +# independently. It cleans out the build output directories, runs a +# specified worker script (an example is test_build_time_worker.sh) under +# TIME(1), logs the results to TEST_LOGDIR (default /tmp) and returns a +# value telling "git bisect run" whether the build time is good (under +# the specified threshold) or bad (over it). There is also a tolerance +# option but it is not particularly useful as it only subtracts the +# tolerance from the given threshold and uses it as the actual threshold. +# +# It is also capable of taking a file listing git revision hashes to be +# test-applied to the repository in order to get past build failures that +# would otherwise cause certain revisions to have to be skipped; if a +# revision does not apply cleanly then the script assumes it does not +# need to be applied and ignores it. +# +# Please see the help output (syntax below) for some important setup +# instructions. +# +# AUTHORS +# Paul Eggleton <paul.eggleton@linux.intel.com> + + +syntax() { + echo "syntax: $0 <script> <time> <tolerance> [patchrevlist]" + echo "" + echo " script - worker script file (if in current dir, prefix with ./)" + echo " time - time threshold (in seconds, suffix m for minutes)" + echo " tolerance - tolerance (in seconds, suffix m for minutes or % for" + echo " percentage, can be 0)" + echo " patchrevlist - optional file listing revisions to apply as patches on top" + echo "" + echo "You must set TEST_BUILDDIR to point to a previously created build directory," + echo "however please note that this script will wipe out the TMPDIR defined in" + echo "TEST_BUILDDIR/conf/local.conf as part of its initial setup (as well as your" + echo "~/.ccache)" + echo "" + echo "To get rid of the sudo prompt, please add the following line to /etc/sudoers" + echo "(use 'visudo' to edit this; also it is assumed that the user you are running" + echo "as is a member of the 'wheel' group):" + echo "" + echo "%wheel ALL=(ALL) NOPASSWD: /sbin/sysctl -w vm.drop_caches=[1-3]" + echo "" + echo "Note: it is recommended that you disable crond and any other process that" + echo "may cause significant CPU or I/O usage during build performance tests." +} + +# Note - we exit with 250 here because that will tell git bisect run that +# something bad happened and stop +if [ "$1" = "" ] ; then + syntax + exit 250 +fi + +if [ "$2" = "" ] ; then + syntax + exit 250 +fi + +if [ "$3" = "" ] ; then + syntax + exit 250 +fi + +if ! [[ "$2" =~ ^[0-9][0-9m.]*$ ]] ; then + echo "'$2' is not a valid number for threshold" + exit 250 +fi + +if ! [[ "$3" =~ ^[0-9][0-9m.%]*$ ]] ; then + echo "'$3' is not a valid number for tolerance" + exit 250 +fi + +if [ "$TEST_BUILDDIR" = "" ] ; then + echo "Please set TEST_BUILDDIR to a previously created build directory" + exit 250 +fi + +if [ ! -d "$TEST_BUILDDIR" ] ; then + echo "TEST_BUILDDIR $TEST_BUILDDIR not found" + exit 250 +fi + +git diff --quiet +if [ $? != 0 ] ; then + echo "Working tree is dirty, cannot proceed" + exit 251 +fi + +if [ "$BB_ENV_EXTRAWHITE" != "" ] ; then + echo "WARNING: you are running after sourcing the build environment script, this is not recommended" +fi + +runscript=$1 +timethreshold=$2 +tolerance=$3 + +if [ "$4" != "" ] ; then + patchrevlist=`cat $4` +else + patchrevlist="" +fi + +if [[ timethreshold == *m* ]] ; then + timethreshold=`echo $timethreshold | sed s/m/*60/ | bc` +fi + +if [[ $tolerance == *m* ]] ; then + tolerance=`echo $tolerance | sed s/m/*60/ | bc` +elif [[ $tolerance == *%* ]] ; then + tolerance=`echo $tolerance | sed s/%//` + tolerance=`echo "scale = 2; (($tolerance * $timethreshold) / 100)" | bc` +fi + +tmpdir=`grep "^TMPDIR" $TEST_BUILDDIR/conf/local.conf | sed -e 's/TMPDIR[ \t]*=[ \t\?]*"//' -e 's/"//'` +if [ "x$tmpdir" = "x" ]; then + echo "Unable to determine TMPDIR from $TEST_BUILDDIR/conf/local.conf, bailing out" + exit 250 +fi +sstatedir=`grep "^SSTATE_DIR" $TEST_BUILDDIR/conf/local.conf | sed -e 's/SSTATE_DIR[ \t\?]*=[ \t]*"//' -e 's/"//'` +if [ "x$sstatedir" = "x" ]; then + echo "Unable to determine SSTATE_DIR from $TEST_BUILDDIR/conf/local.conf, bailing out" + exit 250 +fi + +if [ `expr length $tmpdir` -lt 4 ] ; then + echo "TMPDIR $tmpdir is less than 4 characters, bailing out" + exit 250 +fi + +if [ `expr length $sstatedir` -lt 4 ] ; then + echo "SSTATE_DIR $sstatedir is less than 4 characters, bailing out" + exit 250 +fi + +echo -n "About to wipe out TMPDIR $tmpdir, press Ctrl+C to break out... " +for i in 9 8 7 6 5 4 3 2 1 +do + echo -ne "\x08$i" + sleep 1 +done +echo + +pushd . > /dev/null + +rm -f pseudodone +echo "Removing TMPDIR $tmpdir..." +rm -rf $tmpdir +echo "Removing TMPDIR $tmpdir-*libc..." +rm -rf $tmpdir-*libc +echo "Removing SSTATE_DIR $sstatedir..." +rm -rf $sstatedir +echo "Removing ~/.ccache..." +rm -rf ~/.ccache + +echo "Syncing..." +sync +sync +echo "Dropping VM cache..." +#echo 3 > /proc/sys/vm/drop_caches +sudo /sbin/sysctl -w vm.drop_caches=3 > /dev/null + +if [ "$TEST_LOGDIR" = "" ] ; then + logdir="/tmp" +else + logdir="$TEST_LOGDIR" +fi +rev=`git rev-parse HEAD` +logfile="$logdir/timelog_$rev.log" +echo -n > $logfile + +gitroot=`git rev-parse --show-toplevel` +cd $gitroot +for patchrev in $patchrevlist ; do + echo "Applying $patchrev" + patchfile=`mktemp` + git show $patchrev > $patchfile + git apply --check $patchfile &> /dev/null + if [ $? != 0 ] ; then + echo " ... patch does not apply without errors, ignoring" + else + echo "Applied $patchrev" >> $logfile + git apply $patchfile &> /dev/null + fi + rm $patchfile +done + +sync +echo "Quiescing for 5s..." +sleep 5 + +echo "Running $runscript at $rev..." +timeoutfile=`mktemp` +/usr/bin/time -o $timeoutfile -f "%e\nreal\t%E\nuser\t%Us\nsys\t%Ss\nmaxm\t%Mk" $runscript 2>&1 | tee -a $logfile +exitstatus=$PIPESTATUS + +git reset --hard HEAD > /dev/null +popd > /dev/null + +timeresult=`head -n1 $timeoutfile` +cat $timeoutfile | tee -a $logfile +rm $timeoutfile + +if [ $exitstatus != 0 ] ; then + # Build failed, exit with 125 to tell git bisect run to skip this rev + echo "*** Build failed (exit code $exitstatus), skipping..." | tee -a $logfile + exit 125 +fi + +ret=`echo "scale = 2; $timeresult > $timethreshold - $tolerance" | bc` +echo "Returning $ret" | tee -a $logfile +exit $ret + diff --git a/scripts/contrib/test_build_time_worker.sh b/scripts/contrib/test_build_time_worker.sh new file mode 100755 index 0000000..8e20a9e --- /dev/null +++ b/scripts/contrib/test_build_time_worker.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# This is an example script to be used in conjunction with test_build_time.sh + +if [ "$TEST_BUILDDIR" = "" ] ; then + echo "TEST_BUILDDIR is not set" + exit 1 +fi + +buildsubdir=`basename $TEST_BUILDDIR` +if [ ! -d $buildsubdir ] ; then + echo "Unable to find build subdir $buildsubdir in current directory" + exit 1 +fi + +if [ -f oe-init-build-env ] ; then + . ./oe-init-build-env $buildsubdir +elif [ -f poky-init-build-env ] ; then + . ./poky-init-build-env $buildsubdir +else + echo "Unable to find build environment setup script" + exit 1 +fi + +if [ -f ../meta/recipes-sato/images/core-image-sato.bb ] ; then + target="core-image-sato" +else + target="poky-image-sato" +fi + +echo "Build started at `date "+%Y-%m-%d %H:%M:%S"`" +echo "bitbake $target" +bitbake $target +ret=$? +echo "Build finished at `date "+%Y-%m-%d %H:%M:%S"`" +exit $ret + diff --git a/scripts/contrib/verify-homepage.py b/scripts/contrib/verify-homepage.py new file mode 100755 index 0000000..265ff65 --- /dev/null +++ b/scripts/contrib/verify-homepage.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +# This script can be used to verify HOMEPAGE values for all recipes in +# the current configuration. +# The result is influenced by network environment, since the timeout of connect url is 5 seconds as default. + +import sys +import os +import subprocess +import urllib2 + + +# Allow importing scripts/lib modules +scripts_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/..') +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] +import scriptpath +import scriptutils + +# Allow importing bitbake modules +bitbakepath = scriptpath.add_bitbake_lib_path() + +import bb.tinfoil + +logger = scriptutils.logger_create('verify_homepage') + +def wgetHomepage(pn, homepage): + result = subprocess.call('wget ' + '-q -T 5 -t 1 --spider ' + homepage, shell = True) + if result: + logger.warn("%s: failed to verify HOMEPAGE: %s " % (pn, homepage)) + return 1 + else: + return 0 + +def verifyHomepage(bbhandler): + pkg_pn = bbhandler.cooker.recipecache.pkg_pn + pnlist = sorted(pkg_pn) + count = 0 + checked = [] + for pn in pnlist: + for fn in pkg_pn[pn]: + # There's no point checking multiple BBCLASSEXTENDed variants of the same recipe + realfn, _ = bb.cache.Cache.virtualfn2realfn(fn) + if realfn in checked: + continue + data = bb.cache.Cache.loadDataFull(realfn, bbhandler.cooker.collection.get_file_appends(realfn), bbhandler.config_data) + homepage = data.getVar("HOMEPAGE", True) + if homepage: + try: + urllib2.urlopen(homepage, timeout=5) + except Exception: + count = count + wgetHomepage(os.path.basename(realfn), homepage) + checked.append(realfn) + return count + +if __name__=='__main__': + bbhandler = bb.tinfoil.Tinfoil() + bbhandler.prepare() + logger.info("Start verifying HOMEPAGE:") + failcount = verifyHomepage(bbhandler) + logger.info("Finished verifying HOMEPAGE.") + logger.info("Summary: %s failed" % failcount) diff --git a/scripts/cp-noerror b/scripts/cp-noerror new file mode 100755 index 0000000..28eb90d --- /dev/null +++ b/scripts/cp-noerror @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# +# Allow copying of $1 to $2 but if files in $1 disappear during the copy operation, +# don't error. +# Also don't error if $1 disappears. +# + +import sys +import os +import shutil + +def copytree(src, dst, symlinks=False, ignore=None): + """Based on shutil.copytree""" + names = os.listdir(src) + try: + os.makedirs(dst) + except OSError: + # Already exists + pass + errors = [] + for name in names: + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + d = dstname + if os.path.isdir(dstname): + d = os.path.join(dstname, os.path.basename(srcname)) + if os.path.exists(d): + continue + try: + os.link(srcname, dstname) + except OSError: + shutil.copy2(srcname, dstname) + # catch the Error from the recursive copytree so that we can + # continue with other files + except shutil.Error, err: + errors.extend(err.args[0]) + except EnvironmentError, why: + errors.append((srcname, dstname, str(why))) + try: + shutil.copystat(src, dst) + except OSError, why: + errors.extend((src, dst, str(why))) + if errors: + raise shutil.Error, errors + +try: + copytree(sys.argv[1], sys.argv[2]) +except shutil.Error: + pass +except OSError: + pass diff --git a/scripts/create-pull-request b/scripts/create-pull-request new file mode 100755 index 0000000..479ad6e --- /dev/null +++ b/scripts/create-pull-request @@ -0,0 +1,276 @@ +#!/bin/sh +# +# Copyright (c) 2010-2013, Intel Corporation. +# All Rights Reserved +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See +# the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +# +# This script is intended to be used to prepare a series of patches +# and a cover letter in an appropriate and consistent format for +# submission to Open Embedded and The Yocto Project, as well as to +# related projects and layers. +# + +ODIR=pull-$$ +RELATIVE_TO="master" +COMMIT_ID="HEAD" +PREFIX="PATCH" +RFC=0 + +usage() { +CMD=$(basename $0) +cat <<EOM +Usage: $CMD [-h] [-o output_dir] [-m msg_body_file] [-s subject] [-r relative_to] [-i commit_id] [-d relative_dir] -u remote [-b branch] + -b branch Branch name in the specified remote (default: current branch) + -l local branch Local branch name (default: HEAD) + -c Create an RFC (Request for Comment) patch series + -h Display this help message + -i commit_id Ending commit (default: HEAD) + -m msg_body_file The file containing a blurb to be inserted into the summary email + -o output_dir Specify the output directory for the messages (default: pull-PID) + -p prefix Use [prefix N/M] instead of [PATCH N/M] as the subject prefix + -r relative_to Starting commit (default: master) + -s subject The subject to be inserted into the summary email + -u remote The git remote where the branch is located + -d relative_dir Generate patches relative to directory + + Examples: + $CMD -u contrib -b nitin/basic + $CMD -u contrib -r distro/master -i nitin/distro -b nitin/distro + $CMD -u contrib -r distro/master -i nitin/distro -b nitin/distro -l distro + $CMD -u contrib -r master -i misc -b nitin/misc -o pull-misc + $CMD -u contrib -p "RFC PATCH" -b nitin/experimental + $CMD -u contrib -i misc -b nitin/misc -d ./bitbake +EOM +} + +# Parse and validate arguments +while getopts "b:cd:hi:m:o:p:r:s:u:l:" OPT; do + case $OPT in + b) + BRANCH="$OPTARG" + ;; + l) + L_BRANCH="$OPTARG" + ;; + c) + RFC=1 + ;; + d) + RELDIR="$OPTARG" + ;; + h) + usage + exit 0 + ;; + i) + COMMIT_ID="$OPTARG" + ;; + m) + BODY="$OPTARG" + if [ ! -e "$BODY" ]; then + echo "ERROR: Body file does not exist" + exit 1 + fi + ;; + o) + ODIR="$OPTARG" + ;; + p) + PREFIX="$OPTARG" + ;; + r) + RELATIVE_TO="$OPTARG" + ;; + s) + SUBJECT="$OPTARG" + ;; + u) + REMOTE="$OPTARG" + REMOTE_URL=$(git config remote.$REMOTE.url) + if [ $? -ne 0 ]; then + echo "ERROR: git config failed to find a url for '$REMOTE'" + echo + echo "To add a remote url for $REMOTE, use:" + echo " git config remote.$REMOTE.url <url>" + exit 1 + fi + + # Rewrite private URLs to public URLs + # Determine the repository name for use in the WEB_URL later + case "$REMOTE_URL" in + *@*) + USER_RE="[A-Za-z0-9_.@][A-Za-z0-9_.@-]*\$\?" + PROTO_RE="[a-z][a-z+]*://" + GIT_RE="\(^\($PROTO_RE\)\?$USER_RE@\)\([^:/]*\)[:/]\(.*\)" + REMOTE_URL=${REMOTE_URL%.git} + REMOTE_REPO=$(echo $REMOTE_URL | sed "s#$GIT_RE#\4#") + REMOTE_URL=$(echo $REMOTE_URL | sed "s#$GIT_RE#git://\3/\4#") + ;; + *) + echo "WARNING: Unrecognized remote URL: $REMOTE_URL" + echo " The pull and browse URLs will likely be incorrect" + ;; + esac + ;; + esac +done + +if [ -z "$BRANCH" ]; then + BRANCH=$(git branch | grep -e "^\* " | cut -d' ' -f2) + echo "NOTE: Assuming remote branch '$BRANCH', use -b to override." +fi + +if [ -z "$L_BRANCH" ]; then + L_BRANCH=HEAD + echo "NOTE: Assuming local branch HEAD, use -l to override." +fi + +if [ -z "$REMOTE_URL" ]; then + echo "ERROR: Missing parameter -u, no git remote!" + usage + exit 1 +fi + +if [ $RFC -eq 1 ]; then + PREFIX="RFC $PREFIX" +fi + + +# Set WEB_URL from known remotes +WEB_URL="" +case "$REMOTE_URL" in + *git.yoctoproject.org*) + WEB_URL="http://git.yoctoproject.org/cgit.cgi/$REMOTE_REPO/log/?h=$BRANCH" + ;; + *git.pokylinux.org*) + WEB_URL="http://git.pokylinux.org/cgit.cgi/$REMOTE_REPO/log/?h=$BRANCH" + ;; + *git.openembedded.org*) + WEB_URL="http://cgit.openembedded.org/cgit.cgi/$REMOTE_REPO/log/?h=$BRANCH" + ;; + *github.com*) + WEB_URL="https://github.com/$REMOTE_REPO/tree/$BRANCH" + ;; +esac + +# Perform a sanity test on the web URL. Issue a warning if it is not +# accessible, but do not abort as users may want to run offline. +if [ -n "$WEB_URL" ]; then + wget --no-check-certificate -q $WEB_URL -O /dev/null + if [ $? -ne 0 ]; then + echo "WARNING: Branch '$BRANCH' was not found on the contrib git tree." + echo " Please check your remote and branch parameter before sending." + echo "" + fi +fi + +if [ -e $ODIR ]; then + echo "ERROR: output directory $ODIR exists." + exit 1 +fi +mkdir $ODIR + +if [ -n "$RELDIR" ]; then + ODIR=$(realpath $ODIR) + pdir=$(pwd) + cd $RELDIR + extraopts="--relative" +fi + +# Generate the patches and cover letter +git format-patch $extraopts -M40 --subject-prefix="$PREFIX" -n -o $ODIR --thread=shallow --cover-letter $RELATIVE_TO..$COMMIT_ID > /dev/null + +if [ -z "$(ls -A $ODIR 2> /dev/null)" ]; then + echo "ERROR: $ODIR is empty, no cover letter and patches was generated!" + echo " This is most likely due to that \$RRELATIVE_TO..\$COMMIT_ID" + echo " ($RELATIVE_TO..$COMMIT_ID) don't contain any differences." + rmdir $ODIR + exit 1 +fi + +[ -n "$RELDIR" ] && cd $pdir + +# Customize the cover letter +CL="$ODIR/0000-cover-letter.patch" +PM="$ODIR/pull-msg" +GIT_VERSION=$(`git --version` | tr -d '[:alpha:][:space:].' | sed 's/\(...\).*/\1/') +NEWER_GIT_VERSION=210 +if [ $GIT_VERSION -lt $NEWER_GIT_VERSION ]; then + git request-pull $RELATIVE_TO $REMOTE_URL $COMMIT_ID >> "$PM" +else + git request-pull $RELATIVE_TO $REMOTE_URL $L_BRANCH:$BRANCH >> "$PM" +fi +if [ $? -ne 0 ]; then + echo "ERROR: git request-pull reported an error" + exit 1 +fi + +# The cover letter already has a diffstat, remove it from the pull-msg +# before inserting it. +sed -n "0,\#$REMOTE_URL# p" "$PM" | sed -i "/BLURB HERE/ r /dev/stdin" "$CL" +rm "$PM" + +# If this is an RFC, make that clear in the cover letter +if [ $RFC -eq 1 ]; then +(cat <<EOM +Please review the following changes for suitability for inclusion. If you have +any objections or suggestions for improvement, please respond to the patches. If +you agree with the changes, please provide your Acked-by. + +EOM +) | sed -i "/BLURB HERE/ r /dev/stdin" "$CL" +fi + +# Insert the WEB_URL if there is one +if [ -n "$WEB_URL" ]; then + echo " $WEB_URL" | sed -i "\#$REMOTE_URL# r /dev/stdin" "$CL" +fi + + +# If the user specified a message body, insert it into the cover letter and +# remove the BLURB token. +if [ -n "$BODY" ]; then + sed -i "/BLURB HERE/ r $BODY" "$CL" + sed -i "/BLURB HERE/ d" "$CL" +fi + +# If the user specified a subject, replace the SUBJECT token with it. +if [ -n "$SUBJECT" ]; then + sed -i -e "s/\*\*\* SUBJECT HERE \*\*\*/$SUBJECT/" "$CL" +fi + + +# Generate report for user +cat <<EOM +The following patches have been prepared: +$(for PATCH in $(ls $ODIR/*); do echo " $PATCH"; done) + +Review their content, especially the summary mail: + $CL + +When you are satisfied, you can send them with: + send-pull-request -a -p $ODIR +EOM + +# Check the patches for trailing white space +egrep -q -e "^\+.*\s+$" $ODIR/* +if [ $? -ne 1 ]; then + echo + echo "WARNING: Trailing white space detected at these locations" + egrep -nH --color -e "^\+.*\s+$" $ODIR/* +fi diff --git a/scripts/crosstap b/scripts/crosstap new file mode 100755 index 0000000..58317cf --- /dev/null +++ b/scripts/crosstap @@ -0,0 +1,148 @@ +#!/bin/bash +# +# Run a systemtap script on remote target +# +# Examples (run on build host, target is 192.168.1.xxx): +# $ source oe-init-build-env" +# $ cd ~/my/systemtap/scripts" +# +# $ crosstap root@192.168.1.xxx myscript.stp" +# $ crosstap root@192.168.1.xxx myscript-with-args.stp 99 ninetynine" +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +function usage() { + echo "Usage: $0 <user@hostname> <sytemtap-script> [additional systemtap-script args]" +} + +function setup_usage() { + echo "" + echo "'crosstap' requires a local sdk build of the target system" + echo "(or a build that includes 'tools-profile') in order to build" + echo "kernel modules that can probe the target system." + echo "" + echo "Practically speaking, that means you need to do the following:" + echo " - If you're running a pre-built image, download the release" + echo " and/or BSP tarballs used to build the image." + echo " - If you're working from git sources, just clone the metadata" + echo " and BSP layers needed to build the image you'll be booting." + echo " - Make sure you're properly set up to build a new image (see" + echo " the BSP README and/or the widely available basic documentation" + echo " that discusses how to build images)." + echo " - Build an -sdk version of the image e.g.:" + echo " $ bitbake core-image-sato-sdk" + echo " OR" + echo " - Build a non-sdk image but include the profiling tools:" + echo " [ edit local.conf and add 'tools-profile' to the end of" + echo " the EXTRA_IMAGE_FEATURES variable ]" + echo " $ bitbake core-image-sato" + echo "" + echo " [ NOTE that 'crosstap' needs to be able to ssh into the target" + echo " system, which isn't enabled by default in -minimal images. ]" + echo "" + echo "Once you've build the image on the host system, you're ready to" + echo "boot it (or the equivalent pre-built image) and use 'crosstap'" + echo "to probe it (you need to source the environment as usual first):" + echo "" + echo " $ source oe-init-build-env" + echo " $ cd ~/my/systemtap/scripts" + echo " $ crosstap root@192.168.1.xxx myscript.stp" + echo "" +} + +function systemtap_target_arch() { + SYSTEMTAP_TARGET_ARCH=$1 + case $SYSTEMTAP_TARGET_ARCH in + i?86) + SYSTEMTAP_TARGET_ARCH="i386" + ;; + x86?64*) + SYSTEMTAP_TARGET_ARCH="x86_64" + ;; + arm*) + SYSTEMTAP_TARGET_ARCH="arm" + ;; + powerpc*) + SYSTEMTAP_TARGET_ARCH="powerpc" + ;; + *) + ;; + esac +} + +if [ $# -lt 2 ]; then + usage + exit 1 +fi + +if [ -z "$BUILDDIR" ]; then + echo "Error: Unable to find the BUILDDIR environment variable." + echo "Did you forget to source your build system environment setup script?" + exit 1 +fi + +pushd $PWD +cd $BUILDDIR +BITBAKE_VARS=`bitbake -e virtual/kernel` +popd + +STAGING_BINDIR_TOOLCHAIN=$(echo "$BITBAKE_VARS" | grep ^STAGING_BINDIR_TOOLCHAIN \ + | cut -d '=' -f2 | cut -d '"' -f2) +STAGING_BINDIR_TOOLPREFIX=$(echo "$BITBAKE_VARS" | grep ^TARGET_PREFIX \ + | cut -d '=' -f2 | cut -d '"' -f2) +SYSTEMTAP_HOST_INSTALLDIR=$(echo "$BITBAKE_VARS" | grep ^STAGING_DIR_NATIVE \ + | cut -d '=' -f2 | cut -d '"' -f2) +TARGET_ARCH=$(echo "$BITBAKE_VARS" | grep ^TRANSLATED_TARGET_ARCH \ + | cut -d '=' -f2 | cut -d '"' -f2) +TARGET_KERNEL_BUILDDIR=$(echo "$BITBAKE_VARS" | grep ^B= \ + | cut -d '=' -f2 | cut -d '"' -f2) + +systemtap_target_arch "$TARGET_ARCH" + +if [ ! -d $TARGET_KERNEL_BUILDDIR ] || + [ ! -f $TARGET_KERNEL_BUILDDIR/vmlinux ]; then + echo -e "\nError: No target kernel build found." + echo -e "Did you forget to create a local build of your image?" + setup_usage + exit 1 +fi + +if [ ! -f $SYSTEMTAP_HOST_INSTALLDIR/usr/bin/stap ]; then + echo -e "\nError: Native (host) systemtap not found." + echo -e "Did you accidentally build a local non-sdk image? (or forget to" + echo -e "add 'tools-profile' to EXTRA_IMAGE_FEATURES in your local.conf)?" + setup_usage + exit 1 +fi + +target_user_hostname="$1" +full_script_name="$2" +script_name=$(basename "$2") +script_base=${script_name%.*} +shift 2 + +${SYSTEMTAP_HOST_INSTALLDIR}/usr/bin/stap \ + -a ${SYSTEMTAP_TARGET_ARCH} \ + -B CROSS_COMPILE="${STAGING_BINDIR_TOOLCHAIN}/${STAGING_BINDIR_TOOLPREFIX}" \ + -r ${TARGET_KERNEL_BUILDDIR} \ + -I ${SYSTEMTAP_HOST_INSTALLDIR}/usr/share/systemtap/tapset \ + -R ${SYSTEMTAP_HOST_INSTALLDIR}/usr/share/systemtap/runtime \ + --remote=$target_user_hostname \ + -m $script_base \ + $full_script_name "$@" + +exit 0 diff --git a/scripts/devtool b/scripts/devtool new file mode 100755 index 0000000..9ac6e79 --- /dev/null +++ b/scripts/devtool @@ -0,0 +1,347 @@ +#!/usr/bin/env python + +# OpenEmbedded Development tool +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import os +import argparse +import glob +import re +import ConfigParser +import subprocess +import logging + +basepath = '' +workspace = {} +config = None +context = None + + +scripts_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] +from devtool import DevtoolError, setup_tinfoil +import scriptutils +import argparse_oe +logger = scriptutils.logger_create('devtool') + +plugins = [] + + +class ConfigHandler(object): + config_file = '' + config_obj = None + init_path = '' + workspace_path = '' + + def __init__(self, filename): + self.config_file = filename + self.config_obj = ConfigParser.SafeConfigParser() + + def get(self, section, option, default=None): + try: + ret = self.config_obj.get(section, option) + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + if default != None: + ret = default + else: + raise + return ret + + def read(self): + if os.path.exists(self.config_file): + self.config_obj.read(self.config_file) + + if self.config_obj.has_option('General', 'init_path'): + pth = self.get('General', 'init_path') + self.init_path = os.path.join(basepath, pth) + if not os.path.exists(self.init_path): + logger.error('init_path %s specified in config file cannot be found' % pth) + return False + else: + self.config_obj.add_section('General') + + self.workspace_path = self.get('General', 'workspace_path', os.path.join(basepath, 'workspace')) + return True + + + def write(self): + logger.debug('writing to config file %s' % self.config_file) + self.config_obj.set('General', 'workspace_path', self.workspace_path) + with open(self.config_file, 'w') as f: + self.config_obj.write(f) + + def set(self, section, option, value): + if not self.config_obj.has_section(section): + self.config_obj.add_section(section) + self.config_obj.set(section, option, value) + +class Context: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +def read_workspace(): + global workspace + workspace = {} + if not os.path.exists(os.path.join(config.workspace_path, 'conf', 'layer.conf')): + if context.fixed_setup: + logger.error("workspace layer not set up") + sys.exit(1) + else: + logger.info('Creating workspace layer in %s' % config.workspace_path) + _create_workspace(config.workspace_path, config, basepath) + if not context.fixed_setup: + _enable_workspace_layer(config.workspace_path, config, basepath) + + logger.debug('Reading workspace in %s' % config.workspace_path) + externalsrc_re = re.compile(r'^EXTERNALSRC(_pn-([^ =]+))? *= *"([^"]*)"$') + for fn in glob.glob(os.path.join(config.workspace_path, 'appends', '*.bbappend')): + with open(fn, 'r') as f: + for line in f: + res = externalsrc_re.match(line.rstrip()) + if res: + pn = res.group(2) or os.path.splitext(os.path.basename(fn))[0].split('_')[0] + # Find the recipe file within the workspace, if any + bbfile = os.path.basename(fn).replace('.bbappend', '.bb').replace('%', '*') + recipefile = glob.glob(os.path.join(config.workspace_path, + 'recipes', + pn, + bbfile)) + if recipefile: + recipefile = recipefile[0] + workspace[pn] = {'srctree': res.group(3), + 'bbappend': fn, + 'recipefile': recipefile} + logger.debug('Found recipe %s' % workspace[pn]) + +def create_unlockedsigs(): + """ This function will make unlocked-sigs.inc match the recipes in the + workspace. This runs on every run of devtool, but it lets us ensure + the unlocked items are in sync with the workspace. """ + + confdir = os.path.join(basepath, 'conf') + unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc') + bb.utils.mkdirhier(confdir) + with open(os.path.join(confdir, 'unlocked-sigs.inc'), 'w') as f: + f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" + + "# This layer was created by the OpenEmbedded devtool" + + " utility in order to\n" + + "# contain recipes that are unlocked.\n") + + f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n') + for pn in workspace: + f.write(' ' + pn) + f.write('"') + +def create_workspace(args, config, basepath, workspace): + if args.layerpath: + workspacedir = os.path.abspath(args.layerpath) + else: + workspacedir = os.path.abspath(os.path.join(basepath, 'workspace')) + _create_workspace(workspacedir, config, basepath) + if not args.create_only: + _enable_workspace_layer(workspacedir, config, basepath) + +def _create_workspace(workspacedir, config, basepath): + import bb + + confdir = os.path.join(workspacedir, 'conf') + if os.path.exists(os.path.join(confdir, 'layer.conf')): + logger.info('Specified workspace already set up, leaving as-is') + else: + # Add a config file + bb.utils.mkdirhier(confdir) + with open(os.path.join(confdir, 'layer.conf'), 'w') as f: + f.write('# ### workspace layer auto-generated by devtool ###\n') + f.write('BBPATH =. "$' + '{LAYERDIR}:"\n') + f.write('BBFILES += "$' + '{LAYERDIR}/recipes/*/*.bb \\\n') + f.write(' $' + '{LAYERDIR}/appends/*.bbappend"\n') + f.write('BBFILE_COLLECTIONS += "workspacelayer"\n') + f.write('BBFILE_PATTERN_workspacelayer = "^$' + '{LAYERDIR}/"\n') + f.write('BBFILE_PATTERN_IGNORE_EMPTY_workspacelayer = "1"\n') + f.write('BBFILE_PRIORITY_workspacelayer = "99"\n') + # Add a README file + with open(os.path.join(workspacedir, 'README'), 'w') as f: + f.write('This layer was created by the OpenEmbedded devtool utility in order to\n') + f.write('contain recipes and bbappends. In most instances you should use the\n') + f.write('devtool utility to manage files within it rather than modifying files\n') + f.write('directly (although recipes added with "devtool add" will often need\n') + f.write('direct modification.)\n') + f.write('\nIf you no longer need to use devtool you can remove the path to this\n') + f.write('workspace layer from your conf/bblayers.conf file (and then delete the\n') + f.write('layer, if you wish).\n') + f.write('\nNote that by default, if devtool fetches and unpacks source code, it\n') + f.write('will place it in a subdirectory of a "sources" subdirectory of the\n') + f.write('layer. If you prefer it to be elsewhere you can specify the source\n') + f.write('tree path on the command line.\n') + +def _enable_workspace_layer(workspacedir, config, basepath): + """Ensure the workspace layer is in bblayers.conf""" + import bb + bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf') + if not os.path.exists(bblayers_conf): + logger.error('Unable to find bblayers.conf') + return + _, added = bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, config.workspace_path) + if added: + logger.info('Enabling workspace layer in bblayers.conf') + if config.workspace_path != workspacedir: + # Update our config to point to the new location + config.workspace_path = workspacedir + config.write() + + +def main(): + global basepath + global config + global context + + context = Context(fixed_setup=False) + + # Default basepath + basepath = os.path.dirname(os.path.abspath(__file__)) + + parser = argparse_oe.ArgumentParser(description="OpenEmbedded development tool", + add_help=False, + epilog="Use %(prog)s <subcommand> --help to get help on a specific command") + parser.add_argument('--basepath', help='Base directory of SDK / build directory') + parser.add_argument('--bbpath', help='Explicitly specify the BBPATH, rather than getting it from the metadata') + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') + parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR') + + global_args, unparsed_args = parser.parse_known_args() + + # Help is added here rather than via add_help=True, as we don't want it to + # be handled by parse_known_args() + parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, + help='show this help message and exit') + + if global_args.debug: + logger.setLevel(logging.DEBUG) + elif global_args.quiet: + logger.setLevel(logging.ERROR) + + if global_args.basepath: + # Override + basepath = global_args.basepath + if os.path.exists(os.path.join(basepath, '.devtoolbase')): + context.fixed_setup = True + else: + pth = basepath + while pth != '' and pth != os.sep: + if os.path.exists(os.path.join(pth, '.devtoolbase')): + context.fixed_setup = True + basepath = pth + break + pth = os.path.dirname(pth) + + if not context.fixed_setup: + basepath = os.environ.get('BUILDDIR') + if not basepath: + logger.error("This script can only be run after initialising the build environment (e.g. by using oe-init-build-env)") + sys.exit(1) + + logger.debug('Using basepath %s' % basepath) + + config = ConfigHandler(os.path.join(basepath, 'conf', 'devtool.conf')) + if not config.read(): + return -1 + context.config = config + + bitbake_subdir = config.get('General', 'bitbake_subdir', '') + if bitbake_subdir: + # Normally set for use within the SDK + logger.debug('Using bitbake subdir %s' % bitbake_subdir) + sys.path.insert(0, os.path.join(basepath, bitbake_subdir, 'lib')) + core_meta_subdir = config.get('General', 'core_meta_subdir') + sys.path.insert(0, os.path.join(basepath, core_meta_subdir, 'lib')) + else: + # Standard location + import scriptpath + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + sys.exit(1) + logger.debug('Using standard bitbake path %s' % bitbakepath) + scriptpath.add_oe_lib_path() + + scriptutils.logger_setup_color(logger, global_args.color) + + if global_args.bbpath is None: + tinfoil = setup_tinfoil(config_only=True, basepath=basepath) + global_args.bbpath = tinfoil.config_data.getVar('BBPATH', True) + else: + tinfoil = None + + for path in [scripts_path] + global_args.bbpath.split(':'): + pluginpath = os.path.join(path, 'lib', 'devtool') + scriptutils.load_plugins(logger, plugins, pluginpath) + + if tinfoil: + tinfoil.shutdown() + + subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>') + + subparsers.add_subparser_group('sdk', 'SDK maintenance', -2) + subparsers.add_subparser_group('advanced', 'Advanced', -1) + subparsers.add_subparser_group('starting', 'Beginning work on a recipe', 100) + subparsers.add_subparser_group('info', 'Getting information') + subparsers.add_subparser_group('working', 'Working on a recipe in the workspace') + subparsers.add_subparser_group('testbuild', 'Testing changes on target') + + if not context.fixed_setup: + parser_create_workspace = subparsers.add_parser('create-workspace', + help='Set up workspace in an alternative location', + description='Sets up a new workspace. NOTE: other devtool subcommands will create a workspace automatically as needed, so you only need to use %(prog)s if you want to specify where the workspace should be located.', + group='advanced') + parser_create_workspace.add_argument('layerpath', nargs='?', help='Path in which the workspace layer should be created') + parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace layer, do not alter configuration') + parser_create_workspace.set_defaults(func=create_workspace, no_workspace=True) + + for plugin in plugins: + if hasattr(plugin, 'register_commands'): + plugin.register_commands(subparsers, context) + + args = parser.parse_args(unparsed_args, namespace=global_args) + + if not getattr(args, 'no_workspace', False): + read_workspace() + create_unlockedsigs() + + try: + ret = args.func(args, config, basepath, workspace) + except DevtoolError as err: + if str(err): + logger.error(str(err)) + ret = 1 + except argparse_oe.ArgumentUsageError as ae: + parser.error_subcommand(ae.message, ae.subcommand) + + return ret + + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/scripts/gen-lockedsig-cache b/scripts/gen-lockedsig-cache new file mode 100755 index 0000000..0986a21 --- /dev/null +++ b/scripts/gen-lockedsig-cache @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +import os +import sys +import glob +import shutil +import errno + +def mkdir(d): + try: + os.makedirs(d) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + +if len(sys.argv) < 5: + print("Incorrect number of arguments specified") + print("syntax: gen-lockedsig-cache <locked-sigs.inc> <input-cachedir> <output-cachedir> <nativelsbstring>") + sys.exit(1) + +print('Reading %s' % sys.argv[1]) +sigs = [] +with open(sys.argv[1]) as f: + for l in f.readlines(): + if ":" in l: + sigs.append(l.split(":")[2].split()[0]) + +print('Gathering file list') +files = set() +for s in sigs: + p = sys.argv[2] + "/" + s[:2] + "/*" + s + "*" + files |= set(glob.glob(p)) + p = sys.argv[2] + "/%s/" % sys.argv[4] + s[:2] + "/*" + s + "*" + files |= set(glob.glob(p)) + +print('Processing files') +for f in files: + sys.stdout.write('Processing %s... ' % f) + _, ext = os.path.splitext(f) + if not ext in ['.tgz', '.siginfo', '.sig']: + # Most likely a temp file, skip it + print('skipping') + continue + dst = os.path.join(sys.argv[3], os.path.relpath(f, sys.argv[2])) + destdir = os.path.dirname(dst) + mkdir(destdir) + + if os.path.exists(dst): + os.remove(dst) + if (os.stat(f).st_dev == os.stat(destdir).st_dev): + print('linking') + os.link(f, dst) + else: + print('copying') + shutil.copyfile(f, dst) + +print('Done!') diff --git a/scripts/gen-site-config b/scripts/gen-site-config new file mode 100755 index 0000000..7da7a0b --- /dev/null +++ b/scripts/gen-site-config @@ -0,0 +1,53 @@ +#! /bin/sh +# Copyright (c) 2005-2008 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +cat << EOF +AC_PREREQ(2.57) +AC_INIT([site_wide],[1.0.0]) + +EOF + +# Disable as endian is set in the default config +#echo AC_C_BIGENDIAN +#echo + +if [ -e $1/types ] ; then + while read type ; do + echo "AC_CHECK_SIZEOF([$type])" + done < $1/types + + echo +fi + +if [ -e $1/funcs ]; then + while read func ; do + echo "AC_CHECK_FUNCS([$func])" + done < $1/funcs + + echo +fi + +if [ -e $1/headers ]; then + while read header ; do + echo "AC_CHECK_HEADERS([$header])" + done < $1/headers + + echo +fi + +cat << EOF +AC_OUTPUT +EOF diff --git a/scripts/git-hooks/post-checkout b/scripts/git-hooks/post-checkout new file mode 100755 index 0000000..4dd578c --- /dev/null +++ b/scripts/git-hooks/post-checkout @@ -0,0 +1,3 @@ +#!/bin/bash + +./setup.sh --update diff --git a/scripts/git-hooks/post-merge b/scripts/git-hooks/post-merge new file mode 100755 index 0000000..4dd578c --- /dev/null +++ b/scripts/git-hooks/post-merge @@ -0,0 +1,3 @@ +#!/bin/bash + +./setup.sh --update diff --git a/scripts/git-hooks/pre-commit b/scripts/git-hooks/pre-commit new file mode 100755 index 0000000..ee010cb --- /dev/null +++ b/scripts/git-hooks/pre-commit @@ -0,0 +1,23 @@ +#!/bin/bash + +currentBranch=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') + +if [ "$currentBranch" == "master" ]; then + gitlabOccurances=$(grep -c -m1 gitlab .gitmodules) + + if [ $gitlabOccurances -eq 1 ]; then + echo " +************************************************************************* +COMMIT INVALID: .gitmodules has a reference to GitLab + +You can't commit anything to the master branch that pulls +from GitLab because outsiders won't be able to build it. + +If you need to use a submodule from GitLab, do so from a branch. +*************************************************************************" + exit 1 + fi +fi + +exit 0 # Everything is good + diff --git a/scripts/git-hooks/pre-push b/scripts/git-hooks/pre-push new file mode 100755 index 0000000..9cacb8f --- /dev/null +++ b/scripts/git-hooks/pre-push @@ -0,0 +1,23 @@ +#!/bin/bash + +currentBranch=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') + +if [ "$currentBranch" == "master" ]; then + gitlabOccurances=$(grep -c -m1 gitlab .gitmodules) + + if [ $gitlabOccurances -eq 1 ]; then + echo " +************************************************************************* +PUSH ERROR: .gitmodules has a reference to GitLab + +You can't push anything to the master branch that pulls +from GitLab because outsiders won't be able to build it. + +If you need to use a submodule from GitLab, do so from a branch. +*************************************************************************" + exit 1 + fi +fi + +exit 0 # Everything is good + diff --git a/scripts/lib/argparse_oe.py b/scripts/lib/argparse_oe.py new file mode 100644 index 0000000..bf3ebad --- /dev/null +++ b/scripts/lib/argparse_oe.py @@ -0,0 +1,129 @@ +import sys +import argparse +from collections import defaultdict, OrderedDict + +class ArgumentUsageError(Exception): + """Exception class you can raise (and catch) in order to show the help""" + def __init__(self, message, subcommand=None): + self.message = message + self.subcommand = subcommand + +class ArgumentParser(argparse.ArgumentParser): + """Our own version of argparse's ArgumentParser""" + def __init__(self, *args, **kwargs): + kwargs.setdefault('formatter_class', OeHelpFormatter) + self._subparser_groups = OrderedDict() + super(ArgumentParser, self).__init__(*args, **kwargs) + + def error(self, message): + sys.stderr.write('ERROR: %s\n' % message) + self.print_help() + sys.exit(2) + + def error_subcommand(self, message, subcommand): + if subcommand: + for action in self._actions: + if isinstance(action, argparse._SubParsersAction): + for choice, subparser in action.choices.items(): + if choice == subcommand: + subparser.error(message) + return + self.error(message) + + def add_subparsers(self, *args, **kwargs): + ret = super(ArgumentParser, self).add_subparsers(*args, **kwargs) + # Need a way of accessing the parent parser + ret._parent_parser = self + # Ensure our class gets instantiated + ret._parser_class = ArgumentSubParser + # Hacky way of adding a method to the subparsers object + ret.add_subparser_group = self.add_subparser_group + return ret + + def add_subparser_group(self, groupname, groupdesc, order=0): + self._subparser_groups[groupname] = (groupdesc, order) + + +class ArgumentSubParser(ArgumentParser): + def __init__(self, *args, **kwargs): + if 'group' in kwargs: + self._group = kwargs.pop('group') + if 'order' in kwargs: + self._order = kwargs.pop('order') + super(ArgumentSubParser, self).__init__(*args, **kwargs) + for agroup in self._action_groups: + if agroup.title == 'optional arguments': + agroup.title = 'options' + break + + def parse_known_args(self, args=None, namespace=None): + # This works around argparse not handling optional positional arguments being + # intermixed with other options. A pretty horrible hack, but we're not left + # with much choice given that the bug in argparse exists and it's difficult + # to subclass. + # Borrowed from http://stackoverflow.com/questions/20165843/argparse-how-to-handle-variable-number-of-arguments-nargs + # with an extra workaround (in format_help() below) for the positional + # arguments disappearing from the --help output, as well as structural tweaks. + # Originally simplified from http://bugs.python.org/file30204/test_intermixed.py + positionals = self._get_positional_actions() + for action in positionals: + # deactivate positionals + action.save_nargs = action.nargs + action.nargs = 0 + + namespace, remaining_args = super(ArgumentSubParser, self).parse_known_args(args, namespace) + for action in positionals: + # remove the empty positional values from namespace + if hasattr(namespace, action.dest): + delattr(namespace, action.dest) + for action in positionals: + action.nargs = action.save_nargs + # parse positionals + namespace, extras = super(ArgumentSubParser, self).parse_known_args(remaining_args, namespace) + return namespace, extras + + def format_help(self): + # Quick, restore the positionals! + positionals = self._get_positional_actions() + for action in positionals: + if hasattr(action, 'save_nargs'): + action.nargs = action.save_nargs + return super(ArgumentParser, self).format_help() + + +class OeHelpFormatter(argparse.HelpFormatter): + def _format_action(self, action): + if hasattr(action, '_get_subactions'): + # subcommands list + groupmap = defaultdict(list) + ordermap = {} + subparser_groups = action._parent_parser._subparser_groups + groups = sorted(subparser_groups.keys(), key=lambda item: subparser_groups[item][1], reverse=True) + for subaction in self._iter_indented_subactions(action): + parser = action._name_parser_map[subaction.dest] + group = getattr(parser, '_group', None) + groupmap[group].append(subaction) + if group not in groups: + groups.append(group) + order = getattr(parser, '_order', 0) + ordermap[subaction.dest] = order + + lines = [] + if len(groupmap) > 1: + groupindent = ' ' + else: + groupindent = '' + for group in groups: + subactions = groupmap[group] + if not subactions: + continue + if groupindent: + if not group: + group = 'other' + groupdesc = subparser_groups.get(group, (group, 0))[0] + lines.append(' %s:' % groupdesc) + for subaction in sorted(subactions, key=lambda item: ordermap[item.dest], reverse=True): + lines.append('%s%s' % (groupindent, self._format_action(subaction).rstrip())) + return '\n'.join(lines) + else: + return super(OeHelpFormatter, self)._format_action(action) diff --git a/scripts/lib/bsp/__init__.py b/scripts/lib/bsp/__init__.py new file mode 100644 index 0000000..8bbb6e1 --- /dev/null +++ b/scripts/lib/bsp/__init__.py @@ -0,0 +1,22 @@ +# +# Yocto BSP tools library +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] intel.com> +# diff --git a/scripts/lib/bsp/engine.py b/scripts/lib/bsp/engine.py new file mode 100644 index 0000000..66e2162 --- /dev/null +++ b/scripts/lib/bsp/engine.py @@ -0,0 +1,1947 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This module implements the templating engine used by 'yocto-bsp' to +# create BSPs. The BSP templates are simply the set of files expected +# to appear in a generated BSP, marked up with a small set of tags +# used to customize the output. The engine parses through the +# templates and generates a Python program containing all the logic +# and input elements needed to display and retrieve BSP-specific +# information from the user. The resulting program uses those results +# to generate the final BSP files. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] intel.com> +# + +import os +import sys +from abc import ABCMeta, abstractmethod +from tags import * +import shlex +import json +import subprocess +import shutil + +class Line(): + """ + Generic (abstract) container representing a line that will appear + in the BSP-generating program. + """ + __metaclass__ = ABCMeta + + def __init__(self, line): + self.line = line + self.generated_line = "" + self.prio = sys.maxint + self.discard = False + + @abstractmethod + def gen(self, context = None): + """ + Generate the final executable line that will appear in the + BSP-generation program. + """ + pass + + def escape(self, line): + """ + Escape single and double quotes and backslashes until I find + something better (re.escape() escapes way too much). + """ + return line.replace("\\", "\\\\").replace("\"", "\\\"").replace("'", "\\'") + + def parse_error(self, msg, lineno, line): + raise SyntaxError("%s: %s" % (msg, line)) + + +class NormalLine(Line): + """ + Container for normal (non-tag) lines. + """ + def __init__(self, line): + Line.__init__(self, line) + self.is_filename = False + self.is_dirname = False + self.out_filebase = None + + def gen(self, context = None): + if self.is_filename: + line = "current_file = \"" + os.path.join(self.out_filebase, self.escape(self.line)) + "\"; of = open(current_file, \"w\")" + elif self.is_dirname: + dirname = os.path.join(self.out_filebase, self.escape(self.line)) + line = "if not os.path.exists(\"" + dirname + "\"): os.mkdir(\"" + dirname + "\")" + else: + line = "of.write(\"" + self.escape(self.line) + "\\n\")" + return line + + +class CodeLine(Line): + """ + Container for Python code tag lines. + """ + def __init__(self, line): + Line.__init__(self, line) + + def gen(self, context = None): + return self.line + + +class Assignment: + """ + Representation of everything we know about {{=name }} tags. + Instances of these are used by Assignment lines. + """ + def __init__(self, start, end, name): + self.start = start + self.end = end + self.name = name + + +class AssignmentLine(NormalLine): + """ + Container for normal lines containing assignment tags. Assignment + tags must be in ascending order of 'start' value. + """ + def __init__(self, line): + NormalLine.__init__(self, line) + self.assignments = [] + + def add_assignment(self, start, end, name): + self.assignments.append(Assignment(start, end, name)) + + def gen(self, context = None): + line = self.escape(self.line) + + for assignment in self.assignments: + replacement = "\" + " + assignment.name + " + \"" + idx = line.find(ASSIGN_TAG) + line = line[:idx] + replacement + line[idx + assignment.end - assignment.start:] + if self.is_filename: + return "current_file = \"" + os.path.join(self.out_filebase, line) + "\"; of = open(current_file, \"w\")" + elif self.is_dirname: + dirname = os.path.join(self.out_filebase, line) + return "if not os.path.exists(\"" + dirname + "\"): os.mkdir(\"" + dirname + "\")" + else: + return "of.write(\"" + line + "\\n\")" + + +class InputLine(Line): + """ + Base class for Input lines. + """ + def __init__(self, props, tag, lineno): + Line.__init__(self, tag) + self.props = props + self.lineno = lineno + + try: + self.prio = int(props["prio"]) + except KeyError: + self.prio = sys.maxint + + def gen(self, context = None): + try: + depends_on = self.props["depends-on"] + try: + depends_on_val = self.props["depends-on-val"] + except KeyError: + self.parse_error("No 'depends-on-val' for 'depends-on' property", + self.lineno, self.line) + except KeyError: + pass + + +class EditBoxInputLine(InputLine): + """ + Base class for 'editbox' Input lines. + + props: + name: example - "Load address" + msg: example - "Please enter the load address" + result: + Sets the value of the variable specified by 'name' to + whatever the user typed. + """ + def __init__(self, props, tag, lineno): + InputLine.__init__(self, props, tag, lineno) + + def gen(self, context = None): + InputLine.gen(self, context) + name = self.props["name"] + if not name: + self.parse_error("No input 'name' property found", + self.lineno, self.line) + msg = self.props["msg"] + if not msg: + self.parse_error("No input 'msg' property found", + self.lineno, self.line) + + try: + default_choice = self.props["default"] + except KeyError: + default_choice = "" + + msg += " [default: " + default_choice + "]" + + line = name + " = default(raw_input(\"" + msg + " \"), " + name + ")" + + return line + + +class GitRepoEditBoxInputLine(EditBoxInputLine): + """ + Base class for 'editbox' Input lines for user input of remote git + repos. This class verifies the existence and connectivity of the + specified git repo. + + props: + name: example - "Load address" + msg: example - "Please enter the load address" + result: + Sets the value of the variable specified by 'name' to + whatever the user typed. + """ + def __init__(self, props, tag, lineno): + EditBoxInputLine.__init__(self, props, tag, lineno) + + def gen(self, context = None): + EditBoxInputLine.gen(self, context) + name = self.props["name"] + if not name: + self.parse_error("No input 'name' property found", + self.lineno, self.line) + msg = self.props["msg"] + if not msg: + self.parse_error("No input 'msg' property found", + self.lineno, self.line) + + try: + default_choice = self.props["default"] + except KeyError: + default_choice = "" + + msg += " [default: " + default_choice + "]" + + line = name + " = get_verified_git_repo(\"" + msg + "\"," + name + ")" + + return line + + +class FileEditBoxInputLine(EditBoxInputLine): + """ + Base class for 'editbox' Input lines for user input of existing + files. This class verifies the existence of the specified file. + + props: + name: example - "Load address" + msg: example - "Please enter the load address" + result: + Sets the value of the variable specified by 'name' to + whatever the user typed. + """ + def __init__(self, props, tag, lineno): + EditBoxInputLine.__init__(self, props, tag, lineno) + + def gen(self, context = None): + EditBoxInputLine.gen(self, context) + name = self.props["name"] + if not name: + self.parse_error("No input 'name' property found", + self.lineno, self.line) + msg = self.props["msg"] + if not msg: + self.parse_error("No input 'msg' property found", + self.lineno, self.line) + + try: + default_choice = self.props["default"] + except KeyError: + default_choice = "" + + msg += " [default: " + default_choice + "]" + + line = name + " = get_verified_file(\"" + msg + "\"," + name + ", True)" + + return line + + +class BooleanInputLine(InputLine): + """ + Base class for boolean Input lines. + props: + name: example - "keyboard" + msg: example - "Got keyboard?" + result: + Sets the value of the variable specified by 'name' to "yes" or "no" + example - keyboard = "yes" + """ + def __init__(self, props, tag, lineno): + InputLine.__init__(self, props, tag, lineno) + + def gen(self, context = None): + InputLine.gen(self, context) + name = self.props["name"] + if not name: + self.parse_error("No input 'name' property found", + self.lineno, self.line) + msg = self.props["msg"] + if not msg: + self.parse_error("No input 'msg' property found", + self.lineno, self.line) + + try: + default_choice = self.props["default"] + except KeyError: + default_choice = "" + + msg += " [default: " + default_choice + "]" + + line = name + " = boolean(raw_input(\"" + msg + " \"), " + name + ")" + + return line + + +class ListInputLine(InputLine): + """ + Base class for List-based Input lines. e.g. Choicelist, Checklist. + """ + __metaclass__ = ABCMeta + + def __init__(self, props, tag, lineno): + InputLine.__init__(self, props, tag, lineno) + self.choices = [] + + def gen_choicepair_list(self): + """Generate a list of 2-item val:desc lists from self.choices.""" + if not self.choices: + return None + + choicepair_list = list() + + for choice in self.choices: + choicepair = [] + choicepair.append(choice.val) + choicepair.append(choice.desc) + choicepair_list.append(choicepair) + + return choicepair_list + + def gen_degenerate_choicepair_list(self, choices): + """Generate a list of 2-item val:desc with val=desc from passed-in choices.""" + choicepair_list = list() + + for choice in choices: + choicepair = [] + choicepair.append(choice) + choicepair.append(choice) + choicepair_list.append(choicepair) + + return choicepair_list + + def exec_listgen_fn(self, context = None): + """ + Execute the list-generating function contained as a string in + the "gen" property. + """ + retval = None + try: + fname = self.props["gen"] + modsplit = fname.split('.') + mod_fn = modsplit.pop() + mod = '.'.join(modsplit) + + __import__(mod) + # python 2.7 has a better way to do this using importlib.import_module + m = sys.modules[mod] + + fn = getattr(m, mod_fn) + if not fn: + self.parse_error("couldn't load function specified for 'gen' property ", + self.lineno, self.line) + retval = fn(context) + if not retval: + self.parse_error("function specified for 'gen' property returned nothing ", + self.lineno, self.line) + except KeyError: + pass + + return retval + + def gen_choices_str(self, choicepairs): + """ + Generate a numbered list of choices from a list of choicepairs + for display to the user. + """ + choices_str = "" + + for i, choicepair in enumerate(choicepairs): + choices_str += "\t" + str(i + 1) + ") " + choicepair[1] + "\n" + + return choices_str + + def gen_choices_val_str(self, choicepairs): + """ + Generate an array of choice values corresponding to the + numbered list generated by gen_choices_str(). + """ + choices_val_list = "[" + + for i, choicepair in enumerate(choicepairs): + choices_val_list += "\"" + choicepair[0] + "\"," + choices_val_list += "]" + + return choices_val_list + + def gen_choices_val_list(self, choicepairs): + """ + Generate an array of choice values corresponding to the + numbered list generated by gen_choices_str(). + """ + choices_val_list = [] + + for i, choicepair in enumerate(choicepairs): + choices_val_list.append(choicepair[0]) + + return choices_val_list + + def gen_choices_list(self, context = None, checklist = False): + """ + Generate an array of choice values corresponding to the + numbered list generated by gen_choices_str(). + """ + choices = self.exec_listgen_fn(context) + if choices: + if len(choices) == 0: + self.parse_error("No entries available for input list", + self.lineno, self.line) + choicepairs = self.gen_degenerate_choicepair_list(choices) + else: + if len(self.choices) == 0: + self.parse_error("No entries available for input list", + self.lineno, self.line) + choicepairs = self.gen_choicepair_list() + + return choicepairs + + def gen_choices(self, context = None, checklist = False): + """ + Generate an array of choice values corresponding to the + numbered list generated by gen_choices_str(), display it to + the user, and process the result. + """ + msg = self.props["msg"] + if not msg: + self.parse_error("No input 'msg' property found", + self.lineno, self.line) + + try: + default_choice = self.props["default"] + except KeyError: + default_choice = "" + + msg += " [default: " + default_choice + "]" + + choicepairs = self.gen_choices_list(context, checklist) + + choices_str = self.gen_choices_str(choicepairs) + choices_val_list = self.gen_choices_val_list(choicepairs) + if checklist: + choiceval = default(find_choicevals(raw_input(msg + "\n" + choices_str), choices_val_list), default_choice) + else: + choiceval = default(find_choiceval(raw_input(msg + "\n" + choices_str), choices_val_list), default_choice) + + return choiceval + + +def find_choiceval(choice_str, choice_list): + """ + Take number as string and return val string from choice_list, + empty string if oob. choice_list is a simple python list. + """ + choice_val = "" + + try: + choice_idx = int(choice_str) + if choice_idx <= len(choice_list): + choice_idx -= 1 + choice_val = choice_list[choice_idx] + except ValueError: + pass + + return choice_val + + +def find_choicevals(choice_str, choice_list): + """ + Take numbers as space-separated string and return vals list from + choice_list, empty list if oob. choice_list is a simple python + list. + """ + choice_vals = [] + + choices = choice_str.split() + for choice in choices: + choice_vals.append(find_choiceval(choice, choice_list)) + + return choice_vals + + +def default(input_str, name): + """ + Return default if no input_str, otherwise stripped input_str. + """ + if not input_str: + return name + + return input_str.strip() + + +def verify_git_repo(giturl): + """ + Verify that the giturl passed in can be connected to. This can be + used as a check for the existence of the given repo and/or basic + git remote connectivity. + + Returns True if the connection was successful, fals otherwise + """ + if not giturl: + return False + + gitcmd = "git ls-remote %s > /dev/null 2>&1" % (giturl) + rc = subprocess.call(gitcmd, shell=True) + if rc == 0: + return True + + return False + + +def get_verified_git_repo(input_str, name): + """ + Return git repo if verified, otherwise loop forever asking user + for filename. + """ + msg = input_str.strip() + " " + + giturl = default(raw_input(msg), name) + + while True: + if verify_git_repo(giturl): + return giturl + giturl = default(raw_input(msg), name) + + +def get_verified_file(input_str, name, filename_can_be_null): + """ + Return filename if the file exists, otherwise loop forever asking + user for filename. + """ + msg = input_str.strip() + " " + + filename = default(raw_input(msg), name) + + while True: + if not filename and filename_can_be_null: + return filename + if os.path.isfile(filename): + return filename + filename = default(raw_input(msg), name) + + +def replace_file(replace_this, with_this): + """ + Replace the given file with the contents of filename, retaining + the original filename. + """ + try: + replace_this.close() + shutil.copy(with_this, replace_this.name) + except IOError: + pass + + +def boolean(input_str, name): + """ + Return lowercase version of first char in string, or value in name. + """ + if not input_str: + return name + + str = input_str.lower().strip() + if str and str[0] == "y" or str[0] == "n": + return str[0] + else: + return name + + +def strip_base(input_str): + """ + strip '/base' off the end of input_str, so we can use 'base' in + the branch names we present to the user. + """ + if input_str and input_str.endswith("/base"): + return input_str[:-len("/base")] + return input_str.strip() + + +deferred_choices = {} + +def gen_choices_defer(input_line, context, checklist = False): + """ + Save the context hashed the name of the input item, which will be + passed to the gen function later. + """ + name = input_line.props["name"] + + try: + nameappend = input_line.props["nameappend"] + except KeyError: + nameappend = "" + + try: + branches_base = input_line.props["branches_base"] + except KeyError: + branches_base = "" + + filename = input_line.props["filename"] + + closetag_start = filename.find(CLOSE_TAG) + + if closetag_start != -1: + filename = filename[closetag_start + len(CLOSE_TAG):] + + filename = filename.strip() + filename = os.path.splitext(filename)[0] + + captured_context = capture_context(context) + context["filename"] = filename + captured_context["filename"] = filename + context["nameappend"] = nameappend + captured_context["nameappend"] = nameappend + context["branches_base"] = branches_base + captured_context["branches_base"] = branches_base + + deferred_choice = (input_line, captured_context, checklist) + key = name + "_" + filename + "_" + nameappend + deferred_choices[key] = deferred_choice + + +def invoke_deferred_choices(name): + """ + Invoke the choice generation function using the context hashed by + 'name'. + """ + deferred_choice = deferred_choices[name] + input_line = deferred_choice[0] + context = deferred_choice[1] + checklist = deferred_choice[2] + + context["name"] = name + + choices = input_line.gen_choices(context, checklist) + + return choices + + +class ChoicelistInputLine(ListInputLine): + """ + Base class for choicelist Input lines. + props: + name: example - "xserver_choice" + msg: example - "Please select an xserver for this machine" + result: + Sets the value of the variable specified by 'name' to whichever Choice was chosen + example - xserver_choice = "xserver_vesa" + """ + def __init__(self, props, tag, lineno): + ListInputLine.__init__(self, props, tag, lineno) + + def gen(self, context = None): + InputLine.gen(self, context) + + gen_choices_defer(self, context) + name = self.props["name"] + nameappend = context["nameappend"] + filename = context["filename"] + + try: + default_choice = self.props["default"] + except KeyError: + default_choice = "" + + line = name + " = default(invoke_deferred_choices(\"" + name + "_" + filename + "_" + nameappend + "\"), \"" + default_choice + "\")" + + return line + + +class ListValInputLine(InputLine): + """ + Abstract base class for choice and checkbox Input lines. + """ + def __init__(self, props, tag, lineno): + InputLine.__init__(self, props, tag, lineno) + + try: + self.val = self.props["val"] + except KeyError: + self.parse_error("No input 'val' property found", self.lineno, self.line) + + try: + self.desc = self.props["msg"] + except KeyError: + self.parse_error("No input 'msg' property found", self.lineno, self.line) + + +class ChoiceInputLine(ListValInputLine): + """ + Base class for choicelist item Input lines. + """ + def __init__(self, props, tag, lineno): + ListValInputLine.__init__(self, props, tag, lineno) + + def gen(self, context = None): + return None + + +class ChecklistInputLine(ListInputLine): + """ + Base class for checklist Input lines. + """ + def __init__(self, props, tag, lineno): + ListInputLine.__init__(self, props, tag, lineno) + + def gen(self, context = None): + InputLine.gen(self, context) + + gen_choices_defer(self, context, True) + name = self.props["name"] + nameappend = context["nameappend"] + filename = context["filename"] + + try: + default_choice = self.props["default"] + except KeyError: + default_choice = "" + + line = name + " = default(invoke_deferred_choices(\"" + name + "_" + filename + "_" + nameappend + "\"), \"" + default_choice + "\")" + + return line + + +class CheckInputLine(ListValInputLine): + """ + Base class for checklist item Input lines. + """ + def __init__(self, props, tag, lineno): + ListValInputLine.__init__(self, props, tag, lineno) + + def gen(self, context = None): + return None + + +dirname_substitutions = {} + +class SubstrateBase(object): + """ + Base class for both expanded and unexpanded file and dir container + objects. + """ + def __init__(self, filename, filebase, out_filebase): + self.filename = filename + self.filebase = filebase + self.translated_filename = filename + self.out_filebase = out_filebase + self.raw_lines = [] + self.expanded_lines = [] + self.prev_choicelist = None + + def parse_error(self, msg, lineno, line): + raise SyntaxError("%s: [%s: %d]: %s" % (msg, self.filename, lineno, line)) + + def expand_input_tag(self, tag, lineno): + """ + Input tags consist of the word 'input' at the beginning, + followed by name:value property pairs which are converted into + a dictionary. + """ + propstr = tag[len(INPUT_TAG):] + + props = dict(prop.split(":", 1) for prop in shlex.split(propstr)) + props["filename"] = self.filename + + input_type = props[INPUT_TYPE_PROPERTY] + if not props[INPUT_TYPE_PROPERTY]: + self.parse_error("No input 'type' property found", lineno, tag) + + if input_type == "boolean": + return BooleanInputLine(props, tag, lineno) + if input_type == "edit": + return EditBoxInputLine(props, tag, lineno) + if input_type == "edit-git-repo": + return GitRepoEditBoxInputLine(props, tag, lineno) + if input_type == "edit-file": + return FileEditBoxInputLine(props, tag, lineno) + elif input_type == "choicelist": + self.prev_choicelist = ChoicelistInputLine(props, tag, lineno) + return self.prev_choicelist + elif input_type == "choice": + if not self.prev_choicelist: + self.parse_error("Found 'choice' input tag but no previous choicelist", + lineno, tag) + choice = ChoiceInputLine(props, tag, lineno) + self.prev_choicelist.choices.append(choice) + return choice + elif input_type == "checklist": + return ChecklistInputLine(props, tag, lineno) + elif input_type == "check": + return CheckInputLine(props, tag, lineno) + + def expand_assignment_tag(self, start, line, lineno): + """ + Expand all tags in a line. + """ + expanded_line = AssignmentLine(line.rstrip()) + + while start != -1: + end = line.find(CLOSE_TAG, start) + if end == -1: + self.parse_error("No close tag found for assignment tag", lineno, line) + else: + name = line[start + len(ASSIGN_TAG):end].strip() + expanded_line.add_assignment(start, end + len(CLOSE_TAG), name) + start = line.find(ASSIGN_TAG, end) + + return expanded_line + + def expand_tag(self, line, lineno): + """ + Returns a processed tag line, or None if there was no tag + + The rules for tags are very simple: + - No nested tags + - Tags start with {{ and end with }} + - An assign tag, {{=, can appear anywhere and will + be replaced with what the assignment evaluates to + - Any other tag occupies the whole line it is on + - if there's anything else on the tag line, it's an error + - if it starts with 'input', it's an input tag and + will only be used for prompting and setting variables + - anything else is straight Python + - tags are in effect only until the next blank line or tag or 'pass' tag + - we don't have indentation in tags, but we need some way to end a block + forcefully without blank lines or other tags - that's the 'pass' tag + - todo: implement pass tag + - directories and filenames can have tags as well, but only assignment + and 'if' code lines + - directories and filenames are the only case where normal tags can + coexist with normal text on the same 'line' + """ + start = line.find(ASSIGN_TAG) + if start != -1: + return self.expand_assignment_tag(start, line, lineno) + + start = line.find(OPEN_TAG) + if start == -1: + return None + + end = line.find(CLOSE_TAG, 0) + if end == -1: + self.parse_error("No close tag found for open tag", lineno, line) + + tag = line[start + len(OPEN_TAG):end].strip() + + if not tag.lstrip().startswith(INPUT_TAG): + return CodeLine(tag) + + return self.expand_input_tag(tag, lineno) + + def append_translated_filename(self, filename): + """ + Simply append filename to translated_filename + """ + self.translated_filename = os.path.join(self.translated_filename, filename) + + def get_substituted_file_or_dir_name(self, first_line, tag): + """ + If file or dir names contain name substitutions, return the name + to substitute. Note that this is just the file or dirname and + doesn't include the path. + """ + filename = first_line.find(tag) + if filename != -1: + filename += len(tag) + substituted_filename = first_line[filename:].strip() + this = substituted_filename.find(" this") + if this != -1: + head, tail = os.path.split(self.filename) + substituted_filename = substituted_filename[:this + 1] + tail + if tag == DIRNAME_TAG: # get rid of .noinstall in dirname + substituted_filename = substituted_filename.split('.')[0] + + return substituted_filename + + def get_substituted_filename(self, first_line): + """ + If a filename contains a name substitution, return the name to + substitute. Note that this is just the filename and doesn't + include the path. + """ + return self.get_substituted_file_or_dir_name(first_line, FILENAME_TAG) + + def get_substituted_dirname(self, first_line): + """ + If a dirname contains a name substitution, return the name to + substitute. Note that this is just the dirname and doesn't + include the path. + """ + return self.get_substituted_file_or_dir_name(first_line, DIRNAME_TAG) + + def substitute_filename(self, first_line): + """ + Find the filename in first_line and append it to translated_filename. + """ + substituted_filename = self.get_substituted_filename(first_line) + self.append_translated_filename(substituted_filename); + + def substitute_dirname(self, first_line): + """ + Find the dirname in first_line and append it to translated_filename. + """ + substituted_dirname = self.get_substituted_dirname(first_line) + self.append_translated_filename(substituted_dirname); + + def is_filename_substitution(self, line): + """ + Do we have a filename subustition? + """ + if line.find(FILENAME_TAG) != -1: + return True + return False + + def is_dirname_substitution(self, line): + """ + Do we have a dirname subustition? + """ + if line.find(DIRNAME_TAG) != -1: + return True + return False + + def translate_dirname(self, first_line): + """ + Just save the first_line mapped by filename. The later pass + through the directories will look for a dirname.noinstall + match and grab the substitution line. + """ + dirname_substitutions[self.filename] = first_line + + def translate_dirnames_in_path(self, path): + """ + Translate dirnames below this file or dir, not including tail. + dirname_substititions is keyed on actual untranslated filenames. + translated_path contains the subsititutions for each element. + """ + remainder = path[len(self.filebase)+1:] + translated_path = untranslated_path = self.filebase + + untranslated_dirs = remainder.split(os.sep) + + for dir in untranslated_dirs: + key = os.path.join(untranslated_path, dir + '.noinstall') + try: + first_line = dirname_substitutions[key] + except KeyError: + translated_path = os.path.join(translated_path, dir) + untranslated_path = os.path.join(untranslated_path, dir) + continue + substituted_dir = self.get_substituted_dirname(first_line) + translated_path = os.path.join(translated_path, substituted_dir) + untranslated_path = os.path.join(untranslated_path, dir) + + return translated_path + + def translate_file_or_dir_name(self): + """ + Originally we were allowed to use open/close/assign tags and python + code in the filename, which fit in nicely with the way we + processed the templates and generated code. Now that we can't + do that, we make those tags proper file contents and have this + pass substitute the nice but non-functional names with those + 'strange' ones, and then proceed as usual. + + So, if files or matching dir<.noinstall> files contain + filename substitutions, this function translates them into the + corresponding 'strange' names, which future passes will expand + as they always have. The resulting pathname is kept in the + file or directory's translated_filename. Another way to think + about it is that self.filename is the input filename, and + translated_filename is the output filename before expansion. + """ + # remove leaf file or dirname + head, tail = os.path.split(self.filename) + translated_path = self.translate_dirnames_in_path(head) + self.translated_filename = translated_path + + # This is a dirname - does it have a matching .noinstall with + # a substitution? If so, apply the dirname subsititution. + if not os.path.isfile(self.filename): + key = self.filename + ".noinstall" + try: + first_line = dirname_substitutions[key] + except KeyError: + self.append_translated_filename(tail) + return + self.substitute_dirname(first_line) + return + + f = open(self.filename) + first_line = f.readline() + f.close() + + # This is a normal filename not needing translation, just use + # it as-is. + if not first_line or not first_line.startswith("#"): + self.append_translated_filename(tail) + return + + # If we have a filename substitution (first line in the file + # is a FILENAME_TAG line) do the substitution now. If we have + # a dirname substitution (DIRNAME_TAG in dirname.noinstall + # meta-file), hash it so we can apply it when we see the + # matching dirname later. Otherwise we have a regular + # filename, just use it as-is. + if self.is_filename_substitution(first_line): + self.substitute_filename(first_line) + elif self.is_dirname_substitution(first_line): + self.translate_dirname(first_line) + else: + self.append_translated_filename(tail) + + def expand_file_or_dir_name(self): + """ + Expand file or dir names into codeline. Dirnames and + filenames can only have assignments or if statements. First + translate if statements into CodeLine + (dirname or filename + creation). + """ + lineno = 0 + + line = self.translated_filename[len(self.filebase):] + if line.startswith("/"): + line = line[1:] + opentag_start = -1 + + start = line.find(OPEN_TAG) + while start != -1: + if not line[start:].startswith(ASSIGN_TAG): + opentag_start = start + break + start += len(ASSIGN_TAG) + start = line.find(OPEN_TAG, start) + + if opentag_start != -1: + end = line.find(CLOSE_TAG, opentag_start) + if end == -1: + self.parse_error("No close tag found for open tag", lineno, line) + # we have a {{ tag i.e. code + tag = line[opentag_start + len(OPEN_TAG):end].strip() + if not tag.lstrip().startswith(IF_TAG): + self.parse_error("Only 'if' tags are allowed in file or directory names", + lineno, line) + self.expanded_lines.append(CodeLine(tag)) + + # everything after }} is the actual filename (possibly with assignments) + # everything before is the pathname + line = line[:opentag_start] + line[end + len(CLOSE_TAG):].strip() + + assign_start = line.find(ASSIGN_TAG) + if assign_start != -1: + assignment_tag = self.expand_assignment_tag(assign_start, line, lineno) + if isinstance(self, SubstrateFile): + assignment_tag.is_filename = True + assignment_tag.out_filebase = self.out_filebase + elif isinstance(self, SubstrateDir): + assignment_tag.is_dirname = True + assignment_tag.out_filebase = self.out_filebase + self.expanded_lines.append(assignment_tag) + return + + normal_line = NormalLine(line) + if isinstance(self, SubstrateFile): + normal_line.is_filename = True + normal_line.out_filebase = self.out_filebase + elif isinstance(self, SubstrateDir): + normal_line.is_dirname = True + normal_line.out_filebase = self.out_filebase + self.expanded_lines.append(normal_line) + + def expand(self): + """ + Expand the file or dir name first, eventually this ends up + creating the file or dir. + """ + self.translate_file_or_dir_name() + self.expand_file_or_dir_name() + + +class SubstrateFile(SubstrateBase): + """ + Container for both expanded and unexpanded substrate files. + """ + def __init__(self, filename, filebase, out_filebase): + SubstrateBase.__init__(self, filename, filebase, out_filebase) + + def read(self): + if self.raw_lines: + return + f = open(self.filename) + self.raw_lines = f.readlines() + + def expand(self): + """Expand the contents of all template tags in the file.""" + SubstrateBase.expand(self) + self.read() + + for lineno, line in enumerate(self.raw_lines): + # only first line can be a filename substitition + if lineno == 0 and line.startswith("#") and FILENAME_TAG in line: + continue # skip it - we've already expanded it + expanded_line = self.expand_tag(line, lineno + 1) # humans not 0-based + if not expanded_line: + expanded_line = NormalLine(line.rstrip()) + self.expanded_lines.append(expanded_line) + + def gen(self, context = None): + """Generate the code that generates the BSP.""" + base_indent = 0 + + indent = new_indent = base_indent + + for line in self.expanded_lines: + genline = line.gen(context) + if not genline: + continue + if isinstance(line, InputLine): + line.generated_line = genline + continue + if genline.startswith(OPEN_START): + if indent == 1: + base_indent = 1 + if indent: + if genline == BLANKLINE_STR or (not genline.startswith(NORMAL_START) + and not genline.startswith(OPEN_START)): + indent = new_indent = base_indent + if genline.endswith(":"): + new_indent = base_indent + 1 + line.generated_line = (indent * INDENT_STR) + genline + indent = new_indent + + +class SubstrateDir(SubstrateBase): + """ + Container for both expanded and unexpanded substrate dirs. + """ + def __init__(self, filename, filebase, out_filebase): + SubstrateBase.__init__(self, filename, filebase, out_filebase) + + def expand(self): + SubstrateBase.expand(self) + + def gen(self, context = None): + """Generate the code that generates the BSP.""" + indent = new_indent = 0 + for line in self.expanded_lines: + genline = line.gen(context) + if not genline: + continue + if genline.endswith(":"): + new_indent = 1 + else: + new_indent = 0 + line.generated_line = (indent * INDENT_STR) + genline + indent = new_indent + + +def expand_target(target, all_files, out_filebase): + """ + Expand the contents of all template tags in the target. This + means removing tags and categorizing or creating lines so that + future passes can process and present input lines and generate the + corresponding lines of the Python program that will be exec'ed to + actually produce the final BSP. 'all_files' includes directories. + """ + for root, dirs, files in os.walk(target): + for file in files: + if file.endswith("~") or file.endswith("#"): + continue + f = os.path.join(root, file) + sfile = SubstrateFile(f, target, out_filebase) + sfile.expand() + all_files.append(sfile) + + for dir in dirs: + d = os.path.join(root, dir) + sdir = SubstrateDir(d, target, out_filebase) + sdir.expand() + all_files.append(sdir) + + +def gen_program_machine_lines(machine, program_lines): + """ + Use the input values we got from the command line. + """ + line = "machine = \"" + machine + "\"" + program_lines.append(line) + + line = "layer_name = \"" + machine + "\"" + program_lines.append(line) + + +def sort_inputlines(input_lines): + """Sort input lines according to priority (position).""" + input_lines.sort(key = lambda l: l.prio) + + +def find_parent_dependency(lines, depends_on): + for i, line in lines: + if isinstance(line, CodeLine): + continue + if line.props["name"] == depends_on: + return i + + return -1 + + +def process_inputline_dependencies(input_lines, all_inputlines): + """If any input lines depend on others, put the others first.""" + for line in input_lines: + if isinstance(line, InputLineGroup): + group_inputlines = [] + process_inputline_dependencies(line.group, group_inputlines) + line.group = group_inputlines + all_inputlines.append(line) + continue + + if isinstance(line, CodeLine) or isinstance(line, NormalLine): + all_inputlines.append(line) + continue + + try: + depends_on = line.props["depends-on"] + depends_codeline = "if " + line.props["depends-on"] + " == \"" + line.props["depends-on-val"] + "\":" + all_inputlines.append(CodeLine(depends_codeline)) + all_inputlines.append(line) + except KeyError: + all_inputlines.append(line) + + +def conditional_filename(filename): + """ + Check if the filename itself contains a conditional statement. If + so, return a codeline for it. + """ + opentag_start = filename.find(OPEN_TAG) + + if opentag_start != -1: + if filename[opentag_start:].startswith(ASSIGN_TAG): + return None + end = filename.find(CLOSE_TAG, opentag_start) + if end == -1: + print "No close tag found for open tag in filename %s" % filename + sys.exit(1) + + # we have a {{ tag i.e. code + tag = filename[opentag_start + len(OPEN_TAG):end].strip() + if not tag.lstrip().startswith(IF_TAG): + print "Only 'if' tags are allowed in file or directory names, filename: %s" % filename + sys.exit(1) + + return CodeLine(tag) + + return None + + +class InputLineGroup(InputLine): + """ + InputLine that does nothing but group other input lines + corresponding to all the input lines in a SubstrateFile so they + can be generated as a group. prio is the only property used. + """ + def __init__(self, codeline): + InputLine.__init__(self, {}, "", 0) + self.group = [] + self.prio = sys.maxint + self.group.append(codeline) + + def append(self, line): + self.group.append(line) + if line.prio < self.prio: + self.prio = line.prio + + def len(self): + return len(self.group) + + +def gather_inputlines(files): + """ + Gather all the InputLines - we want to generate them first. + """ + all_inputlines = [] + input_lines = [] + + for file in files: + if isinstance(file, SubstrateFile): + group = None + basename = os.path.basename(file.translated_filename) + + codeline = conditional_filename(basename) + if codeline: + group = InputLineGroup(codeline) + + have_condition = False + condition_to_write = None + for line in file.expanded_lines: + if isinstance(line, CodeLine): + have_condition = True + condition_to_write = line + continue + if isinstance(line, InputLine): + if group: + if condition_to_write: + condition_to_write.prio = line.prio + condition_to_write.discard = True + group.append(condition_to_write) + condition_to_write = None + group.append(line) + else: + if condition_to_write: + condition_to_write.prio = line.prio + condition_to_write.discard = True + input_lines.append(condition_to_write) + condition_to_write = None + input_lines.append(line) + else: + if condition_to_write: + condition_to_write = None + if have_condition: + if not line.line.strip(): + line.discard = True + input_lines.append(line) + have_condition = False + + if group and group.len() > 1: + input_lines.append(group) + + sort_inputlines(input_lines) + process_inputline_dependencies(input_lines, all_inputlines) + + return all_inputlines + + +def run_program_lines(linelist, codedump): + """ + For a single file, print all the python code into a buf and execute it. + """ + buf = "\n".join(linelist) + + if codedump: + of = open("bspgen.out", "w") + of.write(buf) + of.close() + exec buf + + +def gen_target(files, context = None): + """ + Generate the python code for each file. + """ + for file in files: + file.gen(context) + + +def gen_program_header_lines(program_lines): + """ + Generate any imports we need. + """ + program_lines.append("current_file = \"\"") + + +def gen_supplied_property_vals(properties, program_lines): + """ + Generate user-specified entries for input values instead of + generating input prompts. + """ + for name, val in properties.iteritems(): + program_line = name + " = \"" + val + "\"" + program_lines.append(program_line) + + +def gen_initial_property_vals(input_lines, program_lines): + """ + Generate null or default entries for input values, so we don't + have undefined variables. + """ + for line in input_lines: + if isinstance(line, InputLineGroup): + gen_initial_property_vals(line.group, program_lines) + continue + + if isinstance(line, InputLine): + try: + name = line.props["name"] + try: + default_val = "\"" + line.props["default"] + "\"" + except: + default_val = "\"\"" + program_line = name + " = " + default_val + program_lines.append(program_line) + except KeyError: + pass + + +def gen_program_input_lines(input_lines, program_lines, context, in_group = False): + """ + Generate only the input lines used for prompting the user. For + that, we only have input lines and CodeLines that affect the next + input line. + """ + indent = new_indent = 0 + + for line in input_lines: + if isinstance(line, InputLineGroup): + gen_program_input_lines(line.group, program_lines, context, True) + continue + if not line.line.strip(): + continue + + genline = line.gen(context) + if not genline: + continue + if genline.endswith(":"): + new_indent += 1 + else: + if indent > 1 or (not in_group and indent): + new_indent -= 1 + + line.generated_line = (indent * INDENT_STR) + genline + program_lines.append(line.generated_line) + + indent = new_indent + + +def gen_program_lines(target_files, program_lines): + """ + Generate the program lines that make up the BSP generation + program. This appends the generated lines of all target_files to + program_lines, and skips input lines, which are dealt with + separately, or omitted. + """ + for file in target_files: + if file.filename.endswith("noinstall"): + continue + + for line in file.expanded_lines: + if isinstance(line, InputLine): + continue + if line.discard: + continue + + program_lines.append(line.generated_line) + + +def create_context(machine, arch, scripts_path): + """ + Create a context object for use in deferred function invocation. + """ + context = {} + + context["machine"] = machine + context["arch"] = arch + context["scripts_path"] = scripts_path + + return context + + +def capture_context(context): + """ + Create a context object for use in deferred function invocation. + """ + captured_context = {} + + captured_context["machine"] = context["machine"] + captured_context["arch"] = context["arch"] + captured_context["scripts_path"] = context["scripts_path"] + + return captured_context + + +def expand_targets(context, bsp_output_dir, expand_common=True): + """ + Expand all the tags in both the common and machine-specific + 'targets'. + + If expand_common is False, don't expand the common target (this + option is used to create special-purpose layers). + """ + target_files = [] + + machine = context["machine"] + arch = context["arch"] + scripts_path = context["scripts_path"] + + lib_path = scripts_path + '/lib' + bsp_path = lib_path + '/bsp' + arch_path = bsp_path + '/substrate/target/arch' + + if expand_common: + common = os.path.join(arch_path, "common") + expand_target(common, target_files, bsp_output_dir) + + arches = os.listdir(arch_path) + if arch not in arches or arch == "common": + print "Invalid karch, exiting\n" + sys.exit(1) + + target = os.path.join(arch_path, arch) + expand_target(target, target_files, bsp_output_dir) + + gen_target(target_files, context) + + return target_files + + +def yocto_common_create(machine, target, scripts_path, layer_output_dir, codedump, properties_file, properties_str="", expand_common=True): + """ + Common layer-creation code + + machine - user-defined machine name (if needed, will generate 'machine' var) + target - the 'target' the layer will be based on, must be one in + scripts/lib/bsp/substrate/target/arch + scripts_path - absolute path to yocto /scripts dir + layer_output_dir - dirname to create for layer + codedump - dump generated code to bspgen.out + properties_file - use values from this file if nonempty i.e no prompting + properties_str - use values from this string if nonempty i.e no prompting + expand_common - boolean, use the contents of (for bsp layers) arch/common + """ + if os.path.exists(layer_output_dir): + print "\nlayer output dir already exists, exiting. (%s)" % layer_output_dir + sys.exit(1) + + properties = None + + if properties_file: + try: + infile = open(properties_file, "r") + except IOError: + print "Couldn't open properties file %s for reading, exiting" % properties_file + sys.exit(1) + + properties = json.load(infile) + + if properties_str and not properties: + properties = json.loads(properties_str) + + os.mkdir(layer_output_dir) + + context = create_context(machine, target, scripts_path) + target_files = expand_targets(context, layer_output_dir, expand_common) + + input_lines = gather_inputlines(target_files) + + program_lines = [] + + gen_program_header_lines(program_lines) + + gen_initial_property_vals(input_lines, program_lines) + + if properties: + gen_supplied_property_vals(properties, program_lines) + + gen_program_machine_lines(machine, program_lines) + + if not properties: + gen_program_input_lines(input_lines, program_lines, context) + + gen_program_lines(target_files, program_lines) + + run_program_lines(program_lines, codedump) + + +def yocto_layer_create(layer_name, scripts_path, layer_output_dir, codedump, properties_file, properties=""): + """ + Create yocto layer + + layer_name - user-defined layer name + scripts_path - absolute path to yocto /scripts dir + layer_output_dir - dirname to create for layer + codedump - dump generated code to bspgen.out + properties_file - use values from this file if nonempty i.e no prompting + properties - use values from this string if nonempty i.e no prompting + """ + yocto_common_create(layer_name, "layer", scripts_path, layer_output_dir, codedump, properties_file, properties, False) + + print "\nNew layer created in %s.\n" % (layer_output_dir) + print "Don't forget to add it to your BBLAYERS (for details see %s/README)." % (layer_output_dir) + + +def yocto_bsp_create(machine, arch, scripts_path, bsp_output_dir, codedump, properties_file, properties=None): + """ + Create bsp + + machine - user-defined machine name + arch - the arch the bsp will be based on, must be one in + scripts/lib/bsp/substrate/target/arch + scripts_path - absolute path to yocto /scripts dir + bsp_output_dir - dirname to create for BSP + codedump - dump generated code to bspgen.out + properties_file - use values from this file if nonempty i.e no prompting + properties - use values from this string if nonempty i.e no prompting + """ + yocto_common_create(machine, arch, scripts_path, bsp_output_dir, codedump, properties_file, properties) + + print "\nNew %s BSP created in %s" % (arch, bsp_output_dir) + + +def print_dict(items, indent = 0): + """ + Print the values in a possibly nested dictionary. + """ + for key, val in items.iteritems(): + print " "*indent + "\"%s\" :" % key, + if type(val) == dict: + print "{" + print_dict(val, indent + 1) + print " "*indent + "}" + else: + print "%s" % val + + +def get_properties(input_lines): + """ + Get the complete set of properties for all the input items in the + BSP, as a possibly nested dictionary. + """ + properties = {} + + for line in input_lines: + if isinstance(line, InputLineGroup): + statement = line.group[0].line + group_properties = get_properties(line.group) + properties[statement] = group_properties + continue + + if not isinstance(line, InputLine): + continue + + if isinstance(line, ChoiceInputLine): + continue + + props = line.props + item = {} + name = props["name"] + for key, val in props.items(): + if not key == "name": + item[key] = val + properties[name] = item + + return properties + + +def yocto_layer_list_properties(arch, scripts_path, properties_file, expand_common=True): + """ + List the complete set of properties for all the input items in the + layer. If properties_file is non-null, write the complete set of + properties as a nested JSON object corresponding to a possibly + nested dictionary. + """ + context = create_context("unused", arch, scripts_path) + target_files = expand_targets(context, "unused", expand_common) + + input_lines = gather_inputlines(target_files) + + properties = get_properties(input_lines) + if properties_file: + try: + of = open(properties_file, "w") + except IOError: + print "Couldn't open properties file %s for writing, exiting" % properties_file + sys.exit(1) + + json.dump(properties, of, indent=1) + else: + print_dict(properties) + + +def split_nested_property(property): + """ + A property name of the form x.y describes a nested property + i.e. the property y is contained within x and can be addressed + using standard JSON syntax for nested properties. Note that if a + property name itself contains '.', it should be contained in + double quotes. + """ + splittable_property = "" + in_quotes = False + for c in property: + if c == '.' and not in_quotes: + splittable_property += '\n' + continue + if c == '"': + in_quotes = not in_quotes + splittable_property += c + + split_properties = splittable_property.split('\n') + + if len(split_properties) > 1: + return split_properties + + return None + + +def find_input_line_group(substring, input_lines): + """ + Find and return the InputLineGroup containing the specified substring. + """ + for line in input_lines: + if isinstance(line, InputLineGroup): + if substring in line.group[0].line: + return line + + return None + + +def find_input_line(name, input_lines): + """ + Find the input line with the specified name. + """ + for line in input_lines: + if isinstance(line, InputLineGroup): + l = find_input_line(name, line.group) + if l: + return l + + if isinstance(line, InputLine): + try: + if line.props["name"] == name: + return line + if line.props["name"] + "_" + line.props["nameappend"] == name: + return line + except KeyError: + pass + + return None + + +def print_values(type, values_list): + """ + Print the values in the given list of values. + """ + if type == "choicelist": + for value in values_list: + print "[\"%s\", \"%s\"]" % (value[0], value[1]) + elif type == "boolean": + for value in values_list: + print "[\"%s\", \"%s\"]" % (value[0], value[1]) + + +def yocto_layer_list_property_values(arch, property, scripts_path, properties_file, expand_common=True): + """ + List the possible values for a given input property. If + properties_file is non-null, write the complete set of properties + as a JSON object corresponding to an array of possible values. + """ + context = create_context("unused", arch, scripts_path) + context["name"] = property + + target_files = expand_targets(context, "unused", expand_common) + + input_lines = gather_inputlines(target_files) + + properties = get_properties(input_lines) + + nested_properties = split_nested_property(property) + if nested_properties: + # currently the outer property of a nested property always + # corresponds to an input line group + input_line_group = find_input_line_group(nested_properties[0], input_lines) + if input_line_group: + input_lines[:] = input_line_group.group[1:] + # The inner property of a nested property name is the + # actual property name we want, so reset to that + property = nested_properties[1] + + input_line = find_input_line(property, input_lines) + if not input_line: + print "Couldn't find values for property %s" % property + return + + values_list = [] + + type = input_line.props["type"] + if type == "boolean": + values_list.append(["y", "n"]) + elif type == "choicelist" or type == "checklist": + try: + gen_fn = input_line.props["gen"] + if nested_properties: + context["filename"] = nested_properties[0] + try: + context["branches_base"] = input_line.props["branches_base"] + except KeyError: + context["branches_base"] = None + values_list = input_line.gen_choices_list(context, False) + except KeyError: + for choice in input_line.choices: + choicepair = [] + choicepair.append(choice.val) + choicepair.append(choice.desc) + values_list.append(choicepair) + + if properties_file: + try: + of = open(properties_file, "w") + except IOError: + print "Couldn't open properties file %s for writing, exiting" % properties_file + sys.exit(1) + + json.dump(values_list, of) + + print_values(type, values_list) + + +def yocto_bsp_list(args, scripts_path, properties_file): + """ + Print available architectures, or the complete list of properties + defined by the BSP, or the possible values for a particular BSP + property. + """ + if len(args) < 1: + return False + + if args[0] == "karch": + lib_path = scripts_path + '/lib' + bsp_path = lib_path + '/bsp' + arch_path = bsp_path + '/substrate/target/arch' + print "Architectures available:" + for arch in os.listdir(arch_path): + if arch == "common" or arch == "layer": + continue + print " %s" % arch + return True + else: + arch = args[0] + + if len(args) < 2 or len(args) > 3: + return False + + if len(args) == 2: + if args[1] == "properties": + yocto_layer_list_properties(arch, scripts_path, properties_file) + else: + return False + + if len(args) == 3: + if args[1] == "property": + yocto_layer_list_property_values(arch, args[2], scripts_path, properties_file) + else: + return False + + return True + + +def yocto_layer_list(args, scripts_path, properties_file): + """ + Print the complete list of input properties defined by the layer, + or the possible values for a particular layer property. + """ + if len(args) < 1: + return False + + if len(args) < 1 or len(args) > 2: + return False + + if len(args) == 1: + if args[0] == "properties": + yocto_layer_list_properties("layer", scripts_path, properties_file, False) + else: + return False + + if len(args) == 2: + if args[0] == "property": + yocto_layer_list_property_values("layer", args[1], scripts_path, properties_file, False) + else: + return False + + return True + + +def map_standard_kbranch(need_new_kbranch, new_kbranch, existing_kbranch): + """ + Return the linux-yocto bsp branch to use with the specified + kbranch. This handles the -standard variants for 3.4 and 3.8; the + other variants don't need mappings. + """ + if need_new_kbranch == "y": + kbranch = new_kbranch + else: + kbranch = existing_kbranch + + if kbranch.startswith("standard/common-pc-64"): + return "bsp/common-pc-64/common-pc-64-standard.scc" + if kbranch.startswith("standard/common-pc"): + return "bsp/common-pc/common-pc-standard.scc" + else: + return "ktypes/standard/standard.scc" + + +def map_preempt_rt_kbranch(need_new_kbranch, new_kbranch, existing_kbranch): + """ + Return the linux-yocto bsp branch to use with the specified + kbranch. This handles the -preempt-rt variants for 3.4 and 3.8; + the other variants don't need mappings. + """ + if need_new_kbranch == "y": + kbranch = new_kbranch + else: + kbranch = existing_kbranch + + if kbranch.startswith("standard/preempt-rt/common-pc-64"): + return "bsp/common-pc-64/common-pc-64-preempt-rt.scc" + if kbranch.startswith("standard/preempt-rt/common-pc"): + return "bsp/common-pc/common-pc-preempt-rt.scc" + else: + return "ktypes/preempt-rt/preempt-rt.scc" + + +def map_tiny_kbranch(need_new_kbranch, new_kbranch, existing_kbranch): + """ + Return the linux-yocto bsp branch to use with the specified + kbranch. This handles the -tiny variants for 3.4 and 3.8; the + other variants don't need mappings. + """ + if need_new_kbranch == "y": + kbranch = new_kbranch + else: + kbranch = existing_kbranch + + if kbranch.startswith("standard/tiny/common-pc"): + return "bsp/common-pc/common-pc-tiny.scc" + else: + return "ktypes/tiny/tiny.scc" diff --git a/scripts/lib/bsp/help.py b/scripts/lib/bsp/help.py new file mode 100644 index 0000000..85a09dd --- /dev/null +++ b/scripts/lib/bsp/help.py @@ -0,0 +1,1046 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This module implements some basic help invocation functions along +# with the bulk of the help topic text for the Yocto BSP Tools. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] intel.com> +# + +import subprocess +import logging + + +def subcommand_error(args): + logging.info("invalid subcommand %s" % args[0]) + + +def display_help(subcommand, subcommands): + """ + Display help for subcommand. + """ + if subcommand not in subcommands: + return False + + help = subcommands.get(subcommand, subcommand_error)[2] + pager = subprocess.Popen('less', stdin=subprocess.PIPE) + pager.communicate(help) + + return True + + +def yocto_help(args, usage_str, subcommands): + """ + Subcommand help dispatcher. + """ + if len(args) == 1 or not display_help(args[1], subcommands): + print(usage_str) + + +def invoke_subcommand(args, parser, main_command_usage, subcommands): + """ + Dispatch to subcommand handler borrowed from combo-layer. + Should use argparse, but has to work in 2.6. + """ + if not args: + logging.error("No subcommand specified, exiting") + parser.print_help() + elif args[0] == "help": + yocto_help(args, main_command_usage, subcommands) + elif args[0] not in subcommands: + logging.error("Unsupported subcommand %s, exiting\n" % (args[0])) + parser.print_help() + else: + usage = subcommands.get(args[0], subcommand_error)[1] + subcommands.get(args[0], subcommand_error)[0](args[1:], usage) + + +## +# yocto-bsp help and usage strings +## + +yocto_bsp_usage = """ + + Create a customized Yocto BSP layer. + + usage: yocto-bsp [--version] [--help] COMMAND [ARGS] + + Current 'yocto-bsp' commands are: + create Create a new Yocto BSP + list List available values for options and BSP properties + + See 'yocto-bsp help COMMAND' for more information on a specific command. +""" + +yocto_bsp_help_usage = """ + + usage: yocto-bsp help <subcommand> + + This command displays detailed help for the specified subcommand. +""" + +yocto_bsp_create_usage = """ + + Create a new Yocto BSP + + usage: yocto-bsp create <bsp-name> <karch> [-o <DIRNAME> | --outdir <DIRNAME>] + [-i <JSON PROPERTY FILE> | --infile <JSON PROPERTY_FILE>] + [-c | --codedump] [-s | --skip-git-check] + + This command creates a Yocto BSP based on the specified parameters. + The new BSP will be a new Yocto BSP layer contained by default within + the top-level directory specified as 'meta-bsp-name'. The -o option + can be used to place the BSP layer in a directory with a different + name and location. + + The value of the 'karch' parameter determines the set of files that + will be generated for the BSP, along with the specific set of + 'properties' that will be used to fill out the BSP-specific portions + of the BSP. The possible values for the 'karch' parameter can be + listed via 'yocto-bsp list karch'. + + NOTE: Once created, you should add your new layer to your + bblayers.conf file in order for it to be subsequently seen and + modified by the yocto-kernel tool. + + See 'yocto bsp help create' for more detailed instructions. +""" + +yocto_bsp_create_help = """ + +NAME + yocto-bsp create - Create a new Yocto BSP + +SYNOPSIS + yocto-bsp create <bsp-name> <karch> [-o <DIRNAME> | --outdir <DIRNAME>] + [-i <JSON PROPERTY FILE> | --infile <JSON PROPERTY_FILE>] + [-c | --codedump] [-s | --skip-git-check] + +DESCRIPTION + This command creates a Yocto BSP based on the specified + parameters. The new BSP will be a new Yocto BSP layer contained + by default within the top-level directory specified as + 'meta-bsp-name'. The -o option can be used to place the BSP layer + in a directory with a different name and location. + + The value of the 'karch' parameter determines the set of files + that will be generated for the BSP, along with the specific set of + 'properties' that will be used to fill out the BSP-specific + portions of the BSP. The possible values for the 'karch' parameter + can be listed via 'yocto-bsp list karch'. + + The BSP-specific properties that define the values that will be + used to generate a particular BSP can be specified on the + command-line using the -i option and supplying a JSON object + consisting of the set of name:value pairs needed by the BSP. + + If the -i option is not used, the user will be interactively + prompted for each of the required property values, which will then + be used as values for BSP generation. + + The set of properties available for a given architecture can be + listed using the 'yocto-bsp list' command. + + Specifying -c causes the Python code generated and executed to + create the BSP to be dumped to the 'bspgen.out' file in the + current directory, and is useful for debugging. + + NOTE: Once created, you should add your new layer to your + bblayers.conf file in order for it to be subsequently seen and + modified by the yocto-kernel tool. + + For example, assuming your poky repo is at /path/to/poky, your new + BSP layer is at /path/to/poky/meta-mybsp, and your build directory + is /path/to/build: + + $ gedit /path/to/build/conf/bblayers.conf + + BBLAYERS ?= " \\ + /path/to/poky/meta \\ + /path/to/poky/meta-poky \\ + /path/to/poky/meta-mybsp \\ + " +""" + +yocto_bsp_list_usage = """ + + usage: yocto-bsp list karch + yocto-bsp list <karch> properties + [-o <JSON PROPERTY FILE> | --outfile <JSON PROPERTY_FILE>] + yocto-bsp list <karch> property <xxx> + [-o <JSON PROPERTY FILE> | --outfile <JSON PROPERTY_FILE>] + + This command enumerates the complete set of possible values for a + specified option or property needed by the BSP creation process. + + The first form enumerates all the possible values that exist and can + be specified for the 'karch' parameter to the 'yocto bsp create' + command. + + The second form enumerates all the possible properties that exist and + must have values specified for them in the 'yocto bsp create' command + for the given 'karch'. + + The third form enumerates all the possible values that exist and can + be specified for any of the enumerable properties of the given + 'karch' in the 'yocto bsp create' command. + + See 'yocto-bsp help list' for more details. +""" + +yocto_bsp_list_help = """ + +NAME + yocto-bsp list - List available values for options and BSP properties + +SYNOPSIS + yocto-bsp list karch + yocto-bsp list <karch> properties + [--o <JSON PROPERTY FILE> | -outfile <JSON PROPERTY_FILE>] + yocto-bsp list <karch> property <xxx> + [--o <JSON PROPERTY FILE> | -outfile <JSON PROPERTY_FILE>] + +DESCRIPTION + This command enumerates the complete set of possible values for a + specified option or property needed by the BSP creation process. + + The first form enumerates all the possible values that exist and + can be specified for the 'karch' parameter to the 'yocto bsp + create' command. Example output for the 'list karch' command: + + $ yocto-bsp list karch + Architectures available: + arm + powerpc + i386 + mips + mips64 + x86_64 + qemu + + The second form enumerates all the possible properties that exist + and must have values specified for them in the 'yocto bsp create' + command for the given 'karch'. This command is mainly meant to + allow the development user interface alternatives to the default + text-based prompting interface. If the -o option is specified, + the list of properties, in addition to being displayed, will be + written to the specified file as a JSON object. In this case, the + object will consist of the set of name:value pairs corresponding + to the (possibly nested) dictionary of properties defined by the + input statements used by the BSP. Some example output for the + 'list properties' command: + + $ yocto-bsp list arm properties + "touchscreen" : { + "msg" : Does your BSP have a touchscreen? (y/N) + "default" : n + "type" : boolean + } + "uboot_loadaddress" : { + "msg" : Please specify a value for UBOOT_LOADADDRESS. + "default" : 0x80008000 + "type" : edit + "prio" : 40 + } + "kernel_choice" : { + "prio" : 10 + "default" : linux-yocto_3.2 + "depends-on" : use_default_kernel + "depends-on-val" : n + "msg" : Please choose the kernel to use in this BSP => + "type" : choicelist + "gen" : bsp.kernel.kernels + } + "if kernel_choice == "linux-yocto_3.0":" : { + "base_kbranch_linux_yocto_3_0" : { + "prio" : 20 + "default" : yocto/standard + "depends-on" : new_kbranch_linux_yocto_3_0 + "depends-on-val" : y + "msg" : Please choose a machine branch to base this BSP on => + "type" : choicelist + "gen" : bsp.kernel.all_branches + } + . + . + . + + Each entry in the output consists of the name of the input element + e.g. "touchscreen", followed by the properties defined for that + element enclosed in braces. This information should provide + sufficient information to create a complete user interface with. + Two features of the scheme provide for conditional input. First, + if a Python "if" statement appears in place of an input element + name, the set of enclosed input elements apply and should be + presented to the user only if the 'if' statement evaluates to + true. The test in the if statement will always reference another + input element in the list, which means that the element being + tested should be presented to the user before the elements + enclosed by the if block. Secondly, in a similar way, some + elements contain "depends-on" and depends-on-val" tags, which mean + that the affected input element should only be presented to the + user if the element it depends on has already been presented to + the user and the user has selected the specified value for that + element. + + The third form enumerates all the possible values that exist and + can be specified for any of the enumerable properties of the given + 'karch' in the 'yocto bsp create' command. If the -o option is + specified, the list of values for the given property, in addition + to being displayed, will be written to the specified file as a + JSON object. In this case, the object will consist of the set of + name:value pairs corresponding to the array of property values + associated with the property. + + $ yocto-bsp list i386 property xserver_choice + ["xserver_vesa", "VESA xserver support"] + ["xserver_i915", "i915 xserver support"] + + $ yocto-bsp list arm property base_kbranch_linux_yocto_3_0 + Getting branches from remote repo git://git.yoctoproject.org/linux-yocto-3.0... + ["yocto/base", "yocto/base"] + ["yocto/eg20t", "yocto/eg20t"] + ["yocto/gma500", "yocto/gma500"] + ["yocto/pvr", "yocto/pvr"] + ["yocto/standard/arm-versatile-926ejs", "yocto/standard/arm-versatile-926ejs"] + ["yocto/standard/base", "yocto/standard/base"] + ["yocto/standard/cedartrail", "yocto/standard/cedartrail"] + . + . + . + ["yocto/standard/qemu-ppc32", "yocto/standard/qemu-ppc32"] + ["yocto/standard/routerstationpro", "yocto/standard/routerstationpro"] + + The third form as well is meant mainly for developers of + alternative interfaces - it allows the developer to fetch the + possible values for a given input element on-demand. This + on-demand capability is especially valuable for elements that + require relatively expensive remote operations to fulfill, such as + the example that returns the set of branches available in a remote + git tree above. + +""" + +## +# yocto-kernel help and usage strings +## + +yocto_kernel_usage = """ + + Modify and list Yocto BSP kernel config items and patches. + + usage: yocto-kernel [--version] [--help] COMMAND [ARGS] + + Current 'yocto-kernel' commands are: + config list List the modifiable set of bare kernel config options for a BSP + config add Add or modify bare kernel config options for a BSP + config rm Remove bare kernel config options from a BSP + patch list List the patches associated with a BSP + patch add Patch the Yocto kernel for a BSP + patch rm Remove patches from a BSP + feature list List the features used by a BSP + feature add Have a BSP use a feature + feature rm Have a BSP stop using a feature + features list List the features available to BSPs + feature describe Describe a particular feature + feature create Create a new BSP-local feature + feature destroy Remove a BSP-local feature + + See 'yocto-kernel help COMMAND' for more information on a specific command. + +""" + + +yocto_kernel_help_usage = """ + + usage: yocto-kernel help <subcommand> + + This command displays detailed help for the specified subcommand. +""" + +yocto_kernel_config_list_usage = """ + + List the modifiable set of bare kernel config options for a BSP + + usage: yocto-kernel config list <bsp-name> + + This command lists the 'modifiable' config items for a BSP i.e. the + items which are eligible for modification or removal by other + yocto-kernel commands. + + 'modifiable' config items are the config items contained a BSP's + user-config.cfg base config. +""" + + +yocto_kernel_config_list_help = """ + +NAME + yocto-kernel config list - List the modifiable set of bare kernel + config options for a BSP + +SYNOPSIS + yocto-kernel config list <bsp-name> + +DESCRIPTION + This command lists the 'modifiable' config items for a BSP + i.e. the items which are eligible for modification or removal by + other yocto-kernel commands. +""" + + +yocto_kernel_config_add_usage = """ + + Add or modify bare kernel config options for a BSP + + usage: yocto-kernel config add <bsp-name> [<CONFIG_XXX=x> ...] + + This command adds one or more CONFIG_XXX=x items to a BSP's user-config.cfg + base config. +""" + + +yocto_kernel_config_add_help = """ + +NAME + yocto-kernel config add - Add or modify bare kernel config options + for a BSP + +SYNOPSIS + yocto-kernel config add <bsp-name> [<CONFIG_XXX=x> ...] + +DESCRIPTION + This command adds one or more CONFIG_XXX=x items to a BSP's + foo.cfg base config. + + NOTE: It's up to the user to determine whether or not the config + options being added make sense or not - this command does no + sanity checking or verification of any kind to ensure that a + config option really makes sense and will actually be set in in + the final config. For example, if a config option depends on + other config options, it will be turned off by kconfig if the + other options aren't set correctly. +""" + + +yocto_kernel_config_rm_usage = """ + + Remove bare kernel config options from a BSP + + usage: yocto-kernel config rm <bsp-name> + + This command removes (turns off) one or more CONFIG_XXX items from a + BSP's user-config.cfg base config. + + The set of config items available to be removed by this command for a + BSP is listed and the user prompted for the specific items to remove. +""" + + +yocto_kernel_config_rm_help = """ + +NAME + yocto-kernel config rm - Remove bare kernel config options from a + BSP + +SYNOPSIS + yocto-kernel config rm <bsp-name> + +DESCRIPTION + This command removes (turns off) one or more CONFIG_XXX items from a + BSP's user-config.cfg base config. + + The set of config items available to be removed by this command + for a BSP is listed and the user prompted for the specific items + to remove. +""" + + +yocto_kernel_patch_list_usage = """ + + List the patches associated with the kernel for a BSP + + usage: yocto-kernel patch list <bsp-name> + + This command lists the patches associated with a BSP. + + NOTE: this only applies to patches listed in the kernel recipe's + user-patches.scc file (and currently repeated in its SRC_URI). +""" + + +yocto_kernel_patch_list_help = """ + +NAME + yocto-kernel patch list - List the patches associated with the kernel + for a BSP + +SYNOPSIS + yocto-kernel patch list <bsp-name> + +DESCRIPTION + This command lists the patches associated with a BSP. + + NOTE: this only applies to patches listed in the kernel recipe's + user-patches.scc file (and currently repeated in its SRC_URI). +""" + + +yocto_kernel_patch_add_usage = """ + + Patch the Yocto kernel for a specific BSP + + usage: yocto-kernel patch add <bsp-name> [<PATCH> ...] + + This command adds one or more patches to a BSP's machine branch. The + patch will be added to the BSP's linux-yocto kernel user-patches.scc + file (and currently repeated in its SRC_URI) and will be guaranteed + to be applied in the order specified. +""" + + +yocto_kernel_patch_add_help = """ + +NAME + yocto-kernel patch add - Patch the Yocto kernel for a specific BSP + +SYNOPSIS + yocto-kernel patch add <bsp-name> [<PATCH> ...] + +DESCRIPTION + This command adds one or more patches to a BSP's machine branch. + The patch will be added to the BSP's linux-yocto kernel + user-patches.scc file (and currently repeated in its SRC_URI) and + will be guaranteed to be applied in the order specified. + + NOTE: It's up to the user to determine whether or not the patches + being added makes sense or not - this command does no sanity + checking or verification of any kind to ensure that a patch can + actually be applied to the BSP's kernel branch; it's assumed that + the user has already done that. +""" + + +yocto_kernel_patch_rm_usage = """ + + Remove a patch from the Yocto kernel for a specific BSP + + usage: yocto-kernel patch rm <bsp-name> + + This command removes one or more patches from a BSP's machine branch. + The patch will be removed from the BSP's linux-yocto kernel + user-patches.scc file (and currently repeated in its SRC_URI) and + kernel SRC_URI dir. + + The set of patches available to be removed by this command for a BSP + is listed and the user prompted for the specific patches to remove. +""" + + +yocto_kernel_patch_rm_help = """ + +NAME + yocto-kernel patch rm - Remove a patch from the Yocto kernel for a specific BSP + +SYNOPSIS + yocto-kernel patch rm <bsp-name> + +DESCRIPTION + This command removes one or more patches from a BSP's machine + branch. The patch will be removed from the BSP's linux-yocto + kernel user-patches.scc file (and currently repeated in its + SRC_URI). + + The set of patches available to be removed by this command for a + BSP is listed and the user prompted for the specific patches to + remove. +""" + +yocto_kernel_feature_list_usage = """ + + List the BSP features that are being used by a BSP + + usage: yocto-kernel feature list <bsp-name> + + This command lists the features being used by a BSP i.e. the features + which are eligible for modification or removal by other yocto-kernel + commands. + + 'modifiable' features are the features listed in a BSP's + user-features.scc file. +""" + + +yocto_kernel_feature_list_help = """ + +NAME + yocto-kernel feature list - List the modifiable set of features + being used by a BSP + +SYNOPSIS + yocto-kernel feature list <bsp-name> + +DESCRIPTION + This command lists the 'modifiable' features being used by a BSP + i.e. the features which are eligible for modification or removal + by other yocto-kernel commands. +""" + + +yocto_kernel_feature_add_usage = """ + + Add to or modify the list of features being used for a BSP + + usage: yocto-kernel feature add <bsp-name> [/xxxx/yyyy/feature.scc ...] + + This command adds one or more feature items to a BSP's kernel + user-features.scc file, which is the file used to manage features in + a yocto-bsp-generated BSP. Features to be added must be specified as + fully-qualified feature names. +""" + + +yocto_kernel_feature_add_help = """ + +NAME + yocto-kernel feature add - Add to or modify the list of features + being used for a BSP + +SYNOPSIS + yocto-kernel feature add <bsp-name> [/xxxx/yyyy/feature.scc ...] + +DESCRIPTION + This command adds one or more feature items to a BSP's + user-features.scc file, which is the file used to manage features + in a yocto-bsp-generated BSP. Features to be added must be + specified as fully-qualified feature names. +""" + + +yocto_kernel_feature_rm_usage = """ + + Remove a feature from the list of features being used for a BSP + + usage: yocto-kernel feature rm <bsp-name> + + This command removes (turns off) one or more features from a BSP's + user-features.scc file, which is the file used to manage features in + a yocto-bsp-generated BSP. + + The set of features available to be removed by this command for a BSP + is listed and the user prompted for the specific items to remove. +""" + + +yocto_kernel_feature_rm_help = """ + +NAME + yocto-kernel feature rm - Remove a feature from the list of + features being used for a BSP + +SYNOPSIS + yocto-kernel feature rm <bsp-name> + +DESCRIPTION + This command removes (turns off) one or more features from a BSP's + user-features.scc file, which is the file used to manage features + in a yocto-bsp-generated BSP. + + The set of features available to be removed by this command for a + BSP is listed and the user prompted for the specific items to + remove. +""" + + +yocto_kernel_available_features_list_usage = """ + + List the set of kernel features available to a BSP + + usage: yocto-kernel features list <bsp-name> + + This command lists the complete set of kernel features available to a + BSP. This includes the features contained in linux-yocto meta + branches as well as recipe-space features defined locally to the BSP. +""" + + +yocto_kernel_available_features_list_help = """ + +NAME + yocto-kernel features list - List the set of kernel features + available to a BSP + +SYNOPSIS + yocto-kernel features list <bsp-name> + +DESCRIPTION + This command lists the complete set of kernel features available + to a BSP. This includes the features contained in linux-yocto + meta branches as well as recipe-space features defined locally to + the BSP. +""" + + +yocto_kernel_feature_describe_usage = """ + + Print the description and compatibility information for a given kernel feature + + usage: yocto-kernel feature describe <bsp-name> [/xxxx/yyyy/feature.scc ...] + + This command prints the description and compatibility of a specific + feature in the format 'description [compatibility]. +""" + + +yocto_kernel_feature_describe_help = """ + +NAME + yocto-kernel feature describe - print the description and + compatibility information for a given kernel feature + +SYNOPSIS + yocto-kernel feature describe <bsp-name> [/xxxx/yyyy/feature.scc ...] + +DESCRIPTION + This command prints the description and compatibility of a + specific feature in the format 'description [compatibility]. If + the feature doesn't define a description or compatibility, a + string with generic unknown values will be printed. +""" + + +yocto_kernel_feature_create_usage = """ + + Create a recipe-space kernel feature in a BSP + + usage: yocto-kernel feature create <bsp-name> newfeature.scc \ + "Feature Description" capabilities [<CONFIG_XXX=x> ...] [<PATCH> ...] + + This command creates a new kernel feature from the bare config + options and patches specified on the command-line. +""" + + +yocto_kernel_feature_create_help = """ + +NAME + yocto-kernel feature create - create a recipe-space kernel feature + in a BSP + +SYNOPSIS + yocto-kernel feature create <bsp-name> newfeature.scc \ + "Feature Description" capabilities [<CONFIG_XXX=x> ...] [<PATCH> ...] + +DESCRIPTION + This command creates a new kernel feature from the bare config + options and patches specified on the command-line. The new + feature will be created in recipe-space, specifically in either + the kernel .bbappend's /files/cfg or /files/features subdirectory, + depending on whether or not the feature contains config items only + or config items along with patches. The named feature must end + with .scc and must not contain a feature directory to contain the + feature (this will be determined automatically), and a feature + description in double-quotes along with a capabilities string + (which for the time being can be one of: 'all' or 'board'). +""" + + +yocto_kernel_feature_destroy_usage = """ + + Destroy a recipe-space kernel feature in a BSP + + usage: yocto-kernel feature destroy <bsp-name> feature.scc + + This command destroys a kernel feature defined in the specified BSP's + recipe-space kernel definition. +""" + + +yocto_kernel_feature_destroy_help = """ + +NAME + yocto-kernel feature destroy <bsp-name> feature.scc - destroy a + recipe-space kernel feature in a BSP + +SYNOPSIS + yocto-kernel feature destroy <bsp-name> feature.scc + +DESCRIPTION + This command destroys a kernel feature defined in the specified + BSP's recipe-space kernel definition. The named feature must end + with .scc and must not contain a feature directory to contain the + feature (this will be determined automatically). If the kernel + feature is in use by a BSP, it can't be removed until the BSP + stops using it (see yocto-kernel feature rm to stop using it). +""" + +## +# yocto-layer help and usage strings +## + +yocto_layer_usage = """ + + Create a generic Yocto layer. + + usage: yocto-layer [--version] [--help] COMMAND [ARGS] + + Current 'yocto-layer' commands are: + create Create a new generic Yocto layer + list List available values for input options and properties + + See 'yocto-layer help COMMAND' for more information on a specific command. +""" + +yocto_layer_help_usage = """ + + usage: yocto-layer help <subcommand> + + This command displays detailed help for the specified subcommand. +""" + +yocto_layer_create_usage = """ + + Create a new generic Yocto layer + + usage: yocto-layer create <layer-name> [layer_priority] + [-o <DIRNAME> | --outdir <DIRNAME>] + [-i <JSON PROPERTY FILE> | --infile <JSON PROPERTY_FILE>] + + This command creates a generic Yocto layer based on the specified + parameters. The new layer will be a new Yocto layer contained by + default within the top-level directory specified as + 'meta-layer-name'. The -o option can be used to place the layer in a + directory with a different name and location. + + If layer_priority is specified, a simple layer will be created using + the given layer priority, and the user will not be prompted for + further input. + + NOTE: Once created, you should add your new layer to your + bblayers.conf file in order for it to be subsequently seen and + modified by the yocto-kernel tool. Instructions for doing this can + be found in the README file generated in the layer's top-level + directory. + + See 'yocto layer help create' for more detailed instructions. +""" + +yocto_layer_create_help = """ + +NAME + yocto-layer create - Create a new generic Yocto layer + +SYNOPSIS + yocto-layer create <layer-name> [layer_priority] + [-o <DIRNAME> | --outdir <DIRNAME>] + [-i <JSON PROPERTY FILE> | --infile <JSON PROPERTY_FILE>] + +DESCRIPTION + This command creates a generic Yocto layer based on the specified + parameters. The new layer will be a new Yocto layer contained by + default within the top-level directory specified as + 'meta-layer-name'. The -o option can be used to place the layer + in a directory with a different name and location. + + If layer_priority is specified, a simple layer will be created + using the given layer priority, and the user will not be prompted + for further input. + + The layer-specific properties that define the values that will be + used to generate the layer can be specified on the command-line + using the -i option and supplying a JSON object consisting of the + set of name:value pairs needed by the layer. + + If the -i option is not used, the user will be interactively + prompted for each of the required property values, which will then + be used as values for layer generation. + + The set of properties available can be listed using the + 'yocto-layer list' command. + + Specifying -c causes the Python code generated and executed to + create the layer to be dumped to the 'bspgen.out' file in the + current directory, and is useful for debugging. + + NOTE: Once created, you should add your new layer to your + bblayers.conf file in order for it to be subsequently seen and + modified by the yocto-kernel tool. Instructions for doing this + can be found in the README file generated in the layer's top-level + directory. + + For example, assuming your poky repo is at /path/to/poky, your new + layer is at /path/to/poky/meta-mylayer, and your build directory + is /path/to/build: + + $ gedit /path/to/build/conf/bblayers.conf + + BBLAYERS ?= " \\ + /path/to/poky/meta \\ + /path/to/poky/meta-yocto \\ + /path/to/poky/meta-mylayer \\ + " +""" + +yocto_layer_list_usage = """ + + usage: yocto-layer list properties + [-o <JSON PROPERTY FILE> | --outfile <JSON PROPERTY_FILE>] + yocto-layer list property <xxx> + [-o <JSON PROPERTY FILE> | --outfile <JSON PROPERTY_FILE>] + + This command enumerates the complete set of possible values for a + specified option or property needed by the layer creation process. + + The first form enumerates all the possible properties that exist and + must have values specified for them in the 'yocto-layer create' + command. + + The second form enumerates all the possible values that exist and can + be specified for any of the enumerable properties in the 'yocto-layer + create' command. + + See 'yocto-layer help list' for more details. +""" + +yocto_layer_list_help = """ + +NAME + yocto-layer list - List available values for layer input options and properties + +SYNOPSIS + yocto-layer list properties + [--o <JSON PROPERTY FILE> | -outfile <JSON PROPERTY_FILE>] + yocto-layer list property <xxx> + [--o <JSON PROPERTY FILE> | -outfile <JSON PROPERTY_FILE>] + +DESCRIPTION + This command enumerates the complete set of possible values for a + specified option or property needed by the layer creation process. + + The first form enumerates all the possible properties that exist + and must have values specified for them in the 'yocto-layer + create' command. This command is mainly meant to aid the + development of user interface alternatives to the default + text-based prompting interface. If the -o option is specified, + the list of properties, in addition to being displayed, will be + written to the specified file as a JSON object. In this case, the + object will consist of the set of name:value pairs corresponding + to the (possibly nested) dictionary of properties defined by the + input statements used by the BSP. Some example output for the + 'list properties' command: + + $ yocto-layer list properties + "example_bbappend_name" : { + "default" : example + "msg" : Please enter the name you'd like to use for your bbappend file: + "type" : edit + "prio" : 20 + "filename" : /home/trz/yocto/yocto-layer-dev/scripts/lib/bsp/substrate/target/arch/layer/layer-questions.noinstall + } + "create_example_recipe" : { + "default" : n + "msg" : Would you like to have an example recipe created? (y/n) + "type" : boolean + "prio" : 20 + "filename" : /home/trz/yocto/yocto-layer-dev/scripts/lib/bsp/substrate/target/arch/layer/layer-questions.noinstall + } + "example_recipe_name" : { + "default" : example + "msg" : Please enter the name you'd like to use for your example recipe: + "type" : edit + "prio" : 20 + "filename" : /home/trz/yocto/yocto-layer-dev/scripts/lib/bsp/substrate/target/arch/layer/layer-questions.noinstall + } + "layer_priority" : { + "default" : 6 + "msg" : Please enter the layer priority you'd like to use for the layer: + "type" : edit + "prio" : 20 + "filename" : /home/trz/yocto/yocto-layer-dev/scripts/lib/bsp/substrate/target/arch/layer/layer-questions.noinstall + } + "create_example_bbappend" : { + "default" : n + "msg" : Would you like to have an example bbappend file created? (y/n) + "type" : boolean + "prio" : 20 + "filename" : /home/trz/yocto/yocto-layer-dev/scripts/lib/bsp/substrate/target/arch/layer/layer-questions.noinstall + } + "example_bbappend_version" : { + "default" : 0.1 + "msg" : Please enter the version number you'd like to use for your bbappend file (this should match the recipe you're appending to): + "type" : edit + "prio" : 20 + "filename" : /home/trz/yocto/yocto-layer-dev/scripts/lib/bsp/substrate/target/arch/layer/layer-questions.noinstall + } + + Each entry in the output consists of the name of the input element + e.g. "layer_priority", followed by the properties defined for that + element enclosed in braces. This information should provide + sufficient information to create a complete user interface. Two + features of the scheme provide for conditional input. First, if a + Python "if" statement appears in place of an input element name, + the set of enclosed input elements apply and should be presented + to the user only if the 'if' statement evaluates to true. The + test in the if statement will always reference another input + element in the list, which means that the element being tested + should be presented to the user before the elements enclosed by + the if block. Secondly, in a similar way, some elements contain + "depends-on" and depends-on-val" tags, which mean that the + affected input element should only be presented to the user if the + element it depends on has already been presented to the user and + the user has selected the specified value for that element. + + The second form enumerates all the possible values that exist and + can be specified for any of the enumerable properties in the + 'yocto-layer create' command. If the -o option is specified, the + list of values for the given property, in addition to being + displayed, will be written to the specified file as a JSON object. + In this case, the object will consist of the set of name:value + pairs corresponding to the array of property values associated + with the property. + + $ yocto-layer list property layer_priority + [no output - layer_priority is a text field that has no enumerable values] + + The second form as well is meant mainly for developers of + alternative interfaces - it allows the developer to fetch the + possible values for a given input element on-demand. This + on-demand capability is especially valuable for elements that + require relatively expensive remote operations to fulfill, such as + the example that returns the set of branches available in a remote + git tree above. + +""" + +## +# test code +## + +test_bsp_properties = { + 'smp': 'yes', + 'touchscreen': 'yes', + 'keyboard': 'no', + 'xserver': 'yes', + 'xserver_choice': 'xserver-i915', + 'features': ['goodfeature', 'greatfeature'], + 'tunefile': 'tune-quark', +} + diff --git a/scripts/lib/bsp/kernel.py b/scripts/lib/bsp/kernel.py new file mode 100644 index 0000000..0783228 --- /dev/null +++ b/scripts/lib/bsp/kernel.py @@ -0,0 +1,1072 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This module implements the kernel-related functions used by +# 'yocto-kernel' to manage kernel config items and patches for Yocto +# BSPs. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] intel.com> +# + +import sys +import os +import shutil +from tags import * +import glob +import subprocess +from engine import create_context + + +def find_bblayers(): + """ + Find and return a sanitized list of the layers found in BBLAYERS. + """ + try: + builddir = os.environ["BUILDDIR"] + except KeyError: + print "BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)" + sys.exit(1) + bblayers_conf = os.path.join(builddir, "conf/bblayers.conf") + + layers = [] + + bitbake_env_cmd = "bitbake -e" + bitbake_env_lines = subprocess.Popen(bitbake_env_cmd, shell=True, + stdout=subprocess.PIPE).stdout.read() + + if not bitbake_env_lines: + print "Couldn't get '%s' output, exiting." % bitbake_env_cmd + sys.exit(1) + + for line in bitbake_env_lines.split('\n'): + bblayers = get_line_val(line, "BBLAYERS") + if (bblayers): + break + + if not bblayers: + print "Couldn't find BBLAYERS in %s output, exiting." % \ + bitbake_env_cmd + sys.exit(1) + + raw_layers = bblayers.split() + + for layer in raw_layers: + if layer == 'BBLAYERS' or '=' in layer: + continue + layers.append(layer) + + return layers + + +def get_line_val(line, key): + """ + Extract the value from the VAR="val" string + """ + if line.startswith(key + "="): + stripped_line = line.split('=')[1] + stripped_line = stripped_line.replace('\"', '') + return stripped_line + return None + + +def find_meta_layer(): + """ + Find and return the meta layer in BBLAYERS. + """ + layers = find_bblayers() + + for layer in layers: + if layer.endswith("meta"): + return layer + + return None + + +def find_bsp_layer(machine): + """ + Find and return a machine's BSP layer in BBLAYERS. + """ + layers = find_bblayers() + + for layer in layers: + if layer.endswith(machine): + return layer + + print "Unable to find the BSP layer for machine %s." % machine + print "Please make sure it is listed in bblayers.conf" + sys.exit(1) + + +def gen_choices_str(choices): + """ + Generate a numbered list of choices from a list of choices for + display to the user. + """ + choices_str = "" + + for i, choice in enumerate(choices): + choices_str += "\t" + str(i + 1) + ") " + choice + "\n" + + return choices_str + + +def open_user_file(scripts_path, machine, userfile, mode): + """ + Find one of the user files (user-config.cfg, user-patches.scc) + associated with the machine (could be in files/, + linux-yocto-custom/, etc). Returns the open file if found, None + otherwise. + + The caller is responsible for closing the file returned. + """ + layer = find_bsp_layer(machine) + linuxdir = os.path.join(layer, "recipes-kernel/linux") + linuxdir_list = os.listdir(linuxdir) + for fileobj in linuxdir_list: + fileobj_path = os.path.join(linuxdir, fileobj) + if os.path.isdir(fileobj_path): + userfile_name = os.path.join(fileobj_path, userfile) + try: + f = open(userfile_name, mode) + return f + except IOError: + continue + return None + + +def read_config_items(scripts_path, machine): + """ + Find and return a list of config items (CONFIG_XXX) in a machine's + user-defined config fragment [${machine}-user-config.cfg]. + """ + config_items = [] + + f = open_user_file(scripts_path, machine, machine+"-user-config.cfg", "r") + lines = f.readlines() + for line in lines: + s = line.strip() + if s and not s.startswith("#"): + config_items.append(s) + f.close() + + return config_items + + +def write_config_items(scripts_path, machine, config_items): + """ + Write (replace) the list of config items (CONFIG_XXX) in a + machine's user-defined config fragment [${machine}=user-config.cfg]. + """ + f = open_user_file(scripts_path, machine, machine+"-user-config.cfg", "w") + for item in config_items: + f.write(item + "\n") + f.close() + + kernel_contents_changed(scripts_path, machine) + + +def yocto_kernel_config_list(scripts_path, machine): + """ + Display the list of config items (CONFIG_XXX) in a machine's + user-defined config fragment [${machine}-user-config.cfg]. + """ + config_items = read_config_items(scripts_path, machine) + + print "The current set of machine-specific kernel config items for %s is:" % machine + print gen_choices_str(config_items) + + +def yocto_kernel_config_rm(scripts_path, machine): + """ + Display the list of config items (CONFIG_XXX) in a machine's + user-defined config fragment [${machine}-user-config.cfg], prompt the user + for one or more to remove, and remove them. + """ + config_items = read_config_items(scripts_path, machine) + + print "Specify the kernel config items to remove:" + input = raw_input(gen_choices_str(config_items)) + rm_choices = input.split() + rm_choices.sort() + + removed = [] + + for choice in reversed(rm_choices): + try: + idx = int(choice) - 1 + except ValueError: + print "Invalid choice (%s), exiting" % choice + sys.exit(1) + if idx < 0 or idx >= len(config_items): + print "Invalid choice (%d), exiting" % (idx + 1) + sys.exit(1) + removed.append(config_items.pop(idx)) + + write_config_items(scripts_path, machine, config_items) + + print "Removed items:" + for r in removed: + print "\t%s" % r + + +def yocto_kernel_config_add(scripts_path, machine, config_items): + """ + Add one or more config items (CONFIG_XXX) to a machine's + user-defined config fragment [${machine}-user-config.cfg]. + """ + new_items = [] + dup_items = [] + + cur_items = read_config_items(scripts_path, machine) + + for item in config_items: + if not item.startswith("CONFIG") or (not "=y" in item and not "=m" in item): + print "Invalid config item (%s), exiting" % item + sys.exit(1) + if item not in cur_items and item not in new_items: + new_items.append(item) + else: + dup_items.append(item) + + if len(new_items) > 0: + cur_items.extend(new_items) + write_config_items(scripts_path, machine, cur_items) + print "Added item%s:" % ("" if len(new_items)==1 else "s") + for n in new_items: + print "\t%s" % n + + if len(dup_items) > 0: + output="The following item%s already exist%s in the current configuration, ignoring %s:" % \ + (("","s", "it") if len(dup_items)==1 else ("s", "", "them" )) + print output + for n in dup_items: + print "\t%s" % n + +def find_current_kernel(bsp_layer, machine): + """ + Determine the kernel and version currently being used in the BSP. + """ + machine_conf = os.path.join(bsp_layer, "conf/machine/" + machine + ".conf") + + preferred_kernel = preferred_kernel_version = preferred_version_varname = None + + f = open(machine_conf, "r") + lines = f.readlines() + for line in lines: + if line.strip().startswith("PREFERRED_PROVIDER_virtual/kernel"): + preferred_kernel = line.split()[-1] + preferred_kernel = preferred_kernel.replace('\"','') + preferred_version_varname = "PREFERRED_VERSION_" + preferred_kernel + if preferred_version_varname and line.strip().startswith(preferred_version_varname): + preferred_kernel_version = line.split()[-1] + preferred_kernel_version = preferred_kernel_version.replace('\"','') + preferred_kernel_version = preferred_kernel_version.replace('%','') + + if preferred_kernel and preferred_kernel_version: + return preferred_kernel + "_" + preferred_kernel_version + elif preferred_kernel: + return preferred_kernel + + +def find_filesdir(scripts_path, machine): + """ + Find the name of the 'files' dir associated with the machine + (could be in files/, linux-yocto-custom/, etc). Returns the name + of the files dir if found, None otherwise. + """ + layer = find_bsp_layer(machine) + filesdir = None + linuxdir = os.path.join(layer, "recipes-kernel/linux") + linuxdir_list = os.listdir(linuxdir) + for fileobj in linuxdir_list: + fileobj_path = os.path.join(linuxdir, fileobj) + if os.path.isdir(fileobj_path): + # this could be files/ or linux-yocto-custom/, we have no way of distinguishing + # so we take the first (and normally only) dir we find as the 'filesdir' + filesdir = fileobj_path + + return filesdir + + +def read_patch_items(scripts_path, machine): + """ + Find and return a list of patch items in a machine's user-defined + patch list [${machine}-user-patches.scc]. + """ + patch_items = [] + + f = open_user_file(scripts_path, machine, machine+"-user-patches.scc", "r") + lines = f.readlines() + for line in lines: + s = line.strip() + if s and not s.startswith("#"): + fields = s.split() + if not fields[0] == "patch": + continue + patch_items.append(fields[1]) + f.close() + + return patch_items + + +def write_patch_items(scripts_path, machine, patch_items): + """ + Write (replace) the list of patches in a machine's user-defined + patch list [${machine}-user-patches.scc]. + """ + f = open_user_file(scripts_path, machine, machine+"-user-patches.scc", "w") + f.write("mark patching start\n") + for item in patch_items: + f.write("patch " + item + "\n") + f.close() + + kernel_contents_changed(scripts_path, machine) + + +def yocto_kernel_patch_list(scripts_path, machine): + """ + Display the list of patches in a machine's user-defined patch list + [${machine}-user-patches.scc]. + """ + patches = read_patch_items(scripts_path, machine) + + print "The current set of machine-specific patches for %s is:" % machine + print gen_choices_str(patches) + + +def yocto_kernel_patch_rm(scripts_path, machine): + """ + Remove one or more patches from a machine's user-defined patch + list [${machine}-user-patches.scc]. + """ + patches = read_patch_items(scripts_path, machine) + + print "Specify the patches to remove:" + input = raw_input(gen_choices_str(patches)) + rm_choices = input.split() + rm_choices.sort() + + removed = [] + + filesdir = find_filesdir(scripts_path, machine) + if not filesdir: + print "Couldn't rm patch(es) since we couldn't find a 'files' dir" + sys.exit(1) + + for choice in reversed(rm_choices): + try: + idx = int(choice) - 1 + except ValueError: + print "Invalid choice (%s), exiting" % choice + sys.exit(1) + if idx < 0 or idx >= len(patches): + print "Invalid choice (%d), exiting" % (idx + 1) + sys.exit(1) + filesdir_patch = os.path.join(filesdir, patches[idx]) + if os.path.isfile(filesdir_patch): + os.remove(filesdir_patch) + removed.append(patches[idx]) + patches.pop(idx) + + write_patch_items(scripts_path, machine, patches) + + print "Removed patches:" + for r in removed: + print "\t%s" % r + + +def yocto_kernel_patch_add(scripts_path, machine, patches): + """ + Add one or more patches to a machine's user-defined patch list + [${machine}-user-patches.scc]. + """ + existing_patches = read_patch_items(scripts_path, machine) + + for patch in patches: + if os.path.basename(patch) in existing_patches: + print "Couldn't add patch (%s) since it's already been added" % os.path.basename(patch) + sys.exit(1) + + filesdir = find_filesdir(scripts_path, machine) + if not filesdir: + print "Couldn't add patch (%s) since we couldn't find a 'files' dir to add it to" % os.path.basename(patch) + sys.exit(1) + + new_patches = [] + + for patch in patches: + if not os.path.isfile(patch): + print "Couldn't find patch (%s), exiting" % patch + sys.exit(1) + basename = os.path.basename(patch) + filesdir_patch = os.path.join(filesdir, basename) + shutil.copyfile(patch, filesdir_patch) + new_patches.append(basename) + + cur_items = read_patch_items(scripts_path, machine) + cur_items.extend(new_patches) + write_patch_items(scripts_path, machine, cur_items) + + print "Added patches:" + for n in new_patches: + print "\t%s" % n + + +def inc_pr(line): + """ + Add 1 to the PR value in the given bbappend PR line. For the PR + lines in kernel .bbappends after modifications. Handles PRs of + the form PR := "${PR}.1" as well as PR = "r0". + """ + idx = line.find("\"") + + pr_str = line[idx:] + pr_str = pr_str.replace('\"','') + fields = pr_str.split('.') + if len(fields) > 1: + fields[1] = str(int(fields[1]) + 1) + pr_str = "\"" + '.'.join(fields) + "\"\n" + else: + pr_val = pr_str[1:] + pr_str = "\"" + "r" + str(int(pr_val) + 1) + "\"\n" + idx2 = line.find("\"", idx + 1) + line = line[:idx] + pr_str + + return line + + +def kernel_contents_changed(scripts_path, machine): + """ + Do what we need to do to notify the system that the kernel + recipe's contents have changed. + """ + layer = find_bsp_layer(machine) + + kernel = find_current_kernel(layer, machine) + if not kernel: + print "Couldn't determine the kernel for this BSP, exiting." + sys.exit(1) + + kernel_bbfile = os.path.join(layer, "recipes-kernel/linux/" + kernel + ".bbappend") + if not os.path.isfile(kernel_bbfile): + kernel_bbfile = os.path.join(layer, "recipes-kernel/linux/" + kernel + ".bb") + if not os.path.isfile(kernel_bbfile): + return + kernel_bbfile_prev = kernel_bbfile + ".prev" + shutil.copyfile(kernel_bbfile, kernel_bbfile_prev) + + ifile = open(kernel_bbfile_prev, "r") + ofile = open(kernel_bbfile, "w") + ifile_lines = ifile.readlines() + for ifile_line in ifile_lines: + if ifile_line.strip().startswith("PR"): + ifile_line = inc_pr(ifile_line) + ofile.write(ifile_line) + ofile.close() + ifile.close() + + +def kernels(context): + """ + Return the list of available kernels in the BSP i.e. corresponding + to the kernel .bbappends found in the layer. + """ + archdir = os.path.join(context["scripts_path"], "lib/bsp/substrate/target/arch/" + context["arch"]) + kerndir = os.path.join(archdir, "recipes-kernel/linux") + bbglob = os.path.join(kerndir, "*.bbappend") + + bbappends = glob.glob(bbglob) + + kernels = [] + + for kernel in bbappends: + filename = os.path.splitext(os.path.basename(kernel))[0] + idx = filename.find(CLOSE_TAG) + if idx != -1: + filename = filename[idx + len(CLOSE_TAG):].strip() + kernels.append(filename) + + kernels.append("custom") + + return kernels + + +def extract_giturl(file): + """ + Extract the git url of the kernel repo from the kernel recipe's + SRC_URI. + """ + url = None + f = open(file, "r") + lines = f.readlines() + for line in lines: + line = line.strip() + if line.startswith("SRC_URI"): + line = line[len("SRC_URI"):].strip() + if line.startswith("="): + line = line[1:].strip() + if line.startswith("\""): + line = line[1:].strip() + prot = "git" + for s in line.split(";"): + if s.startswith("git://"): + url = s + if s.startswith("protocol="): + prot = s.split("=")[1] + if url: + url = prot + url[3:] + return url + + +def find_giturl(context): + """ + Find the git url of the kernel repo from the kernel recipe's + SRC_URI. + """ + name = context["name"] + filebase = context["filename"] + scripts_path = context["scripts_path"] + + meta_layer = find_meta_layer() + + kerndir = os.path.join(meta_layer, "recipes-kernel/linux") + bbglob = os.path.join(kerndir, "*.bb") + bbs = glob.glob(bbglob) + for kernel in bbs: + filename = os.path.splitext(os.path.basename(kernel))[0] + if filename in filebase: + giturl = extract_giturl(kernel) + return giturl + + return None + + +def read_features(scripts_path, machine): + """ + Find and return a list of features in a machine's user-defined + features fragment [${machine}-user-features.scc]. + """ + features = [] + + f = open_user_file(scripts_path, machine, machine+"-user-features.scc", "r") + lines = f.readlines() + for line in lines: + s = line.strip() + if s and not s.startswith("#"): + feature_include = s.split() + features.append(feature_include[1].strip()) + f.close() + + return features + + +def write_features(scripts_path, machine, features): + """ + Write (replace) the list of feature items in a + machine's user-defined features fragment [${machine}=user-features.cfg]. + """ + f = open_user_file(scripts_path, machine, machine+"-user-features.scc", "w") + for item in features: + f.write("include " + item + "\n") + f.close() + + kernel_contents_changed(scripts_path, machine) + + +def yocto_kernel_feature_list(scripts_path, machine): + """ + Display the list of features used in a machine's user-defined + features fragment [${machine}-user-features.scc]. + """ + features = read_features(scripts_path, machine) + + print "The current set of machine-specific features for %s is:" % machine + print gen_choices_str(features) + + +def yocto_kernel_feature_rm(scripts_path, machine): + """ + Display the list of features used in a machine's user-defined + features fragment [${machine}-user-features.scc], prompt the user + for one or more to remove, and remove them. + """ + features = read_features(scripts_path, machine) + + print "Specify the features to remove:" + input = raw_input(gen_choices_str(features)) + rm_choices = input.split() + rm_choices.sort() + + removed = [] + + for choice in reversed(rm_choices): + try: + idx = int(choice) - 1 + except ValueError: + print "Invalid choice (%s), exiting" % choice + sys.exit(1) + if idx < 0 or idx >= len(features): + print "Invalid choice (%d), exiting" % (idx + 1) + sys.exit(1) + removed.append(features.pop(idx)) + + write_features(scripts_path, machine, features) + + print "Removed features:" + for r in removed: + print "\t%s" % r + + +def yocto_kernel_feature_add(scripts_path, machine, features): + """ + Add one or more features a machine's user-defined features + fragment [${machine}-user-features.scc]. + """ + new_items = [] + + for item in features: + if not item.endswith(".scc"): + print "Invalid feature (%s), exiting" % item + sys.exit(1) + new_items.append(item) + + cur_items = read_features(scripts_path, machine) + cur_items.extend(new_items) + + write_features(scripts_path, machine, cur_items) + + print "Added features:" + for n in new_items: + print "\t%s" % n + + +def find_feature_url(git_url): + """ + Find the url of the kern-features.rc kernel for the kernel repo + specified from the BSP's kernel recipe SRC_URI. + """ + feature_url = "" + if git_url.startswith("git://"): + git_url = git_url[len("git://"):].strip() + s = git_url.split("/") + if s[1].endswith(".git"): + s[1] = s[1][:len(s[1]) - len(".git")] + feature_url = "http://" + s[0] + "/cgit/cgit.cgi/" + s[1] + \ + "/plain/meta/cfg/kern-features.rc?h=meta" + + return feature_url + + +def find_feature_desc(lines): + """ + Find the feature description and compatibility in the passed-in + set of lines. Returns a string string of the form 'desc + [compat]'. + """ + desc = "no description available" + compat = "unknown" + + for line in lines: + idx = line.find("KFEATURE_DESCRIPTION") + if idx != -1: + desc = line[idx + len("KFEATURE_DESCRIPTION"):].strip() + if desc.startswith("\""): + desc = desc[1:] + if desc.endswith("\""): + desc = desc[:-1] + else: + idx = line.find("KFEATURE_COMPATIBILITY") + if idx != -1: + compat = line[idx + len("KFEATURE_COMPATIBILITY"):].strip() + + return desc + " [" + compat + "]" + + +def print_feature_descs(layer, feature_dir): + """ + Print the feature descriptions for the features in feature_dir. + """ + kernel_files_features = os.path.join(layer, "recipes-kernel/linux/files/" + + feature_dir) + for root, dirs, files in os.walk(kernel_files_features): + for file in files: + if file.endswith("~") or file.endswith("#"): + continue + if file.endswith(".scc"): + fullpath = os.path.join(layer, "recipes-kernel/linux/files/" + + feature_dir + "/" + file) + f = open(fullpath) + feature_desc = find_feature_desc(f.readlines()) + print feature_dir + "/" + file + ": " + feature_desc + + +def yocto_kernel_available_features_list(scripts_path, machine): + """ + Display the list of all the kernel features available for use in + BSPs, as gathered from the set of feature sources. + """ + layer = find_bsp_layer(machine) + kernel = find_current_kernel(layer, machine) + if not kernel: + print "Couldn't determine the kernel for this BSP, exiting." + sys.exit(1) + + context = create_context(machine, "arch", scripts_path) + context["name"] = "name" + context["filename"] = kernel + giturl = find_giturl(context) + feature_url = find_feature_url(giturl) + + feature_cmd = "wget -q -O - " + feature_url + tmp = subprocess.Popen(feature_cmd, shell=True, stdout=subprocess.PIPE).stdout.read() + + print "The current set of kernel features available to %s is:\n" % machine + + if tmp: + tmpline = tmp.split("\n") + in_kernel_options = False + for line in tmpline: + if not "=" in line: + if in_kernel_options: + break + if "kernel-options" in line: + in_kernel_options = True + continue + if in_kernel_options: + feature_def = line.split("=") + feature_type = feature_def[0].strip() + feature = feature_def[1].strip() + desc = get_feature_desc(giturl, feature) + print "%s: %s" % (feature, desc) + + print "[local]" + + print_feature_descs(layer, "cfg") + print_feature_descs(layer, "features") + + +def find_feature_desc_url(git_url, feature): + """ + Find the url of the kernel feature in the kernel repo specified + from the BSP's kernel recipe SRC_URI. + """ + feature_desc_url = "" + if git_url.startswith("git://"): + git_url = git_url[len("git://"):].strip() + s = git_url.split("/") + if s[1].endswith(".git"): + s[1] = s[1][:len(s[1]) - len(".git")] + feature_desc_url = "http://" + s[0] + "/cgit/cgit.cgi/" + s[1] + \ + "/plain/meta/cfg/kernel-cache/" + feature + "?h=meta" + + return feature_desc_url + + +def get_feature_desc(git_url, feature): + """ + Return a feature description of the form 'description [compatibility] + BSPs, as gathered from the set of feature sources. + """ + feature_desc_url = find_feature_desc_url(git_url, feature) + feature_desc_cmd = "wget -q -O - " + feature_desc_url + tmp = subprocess.Popen(feature_desc_cmd, shell=True, stdout=subprocess.PIPE).stdout.read() + + return find_feature_desc(tmp.split("\n")) + + +def yocto_kernel_feature_describe(scripts_path, machine, feature): + """ + Display the description of a specific kernel feature available for + use in a BSP. + """ + layer = find_bsp_layer(machine) + + kernel = find_current_kernel(layer, machine) + if not kernel: + print "Couldn't determine the kernel for this BSP, exiting." + sys.exit(1) + + context = create_context(machine, "arch", scripts_path) + context["name"] = "name" + context["filename"] = kernel + giturl = find_giturl(context) + + desc = get_feature_desc(giturl, feature) + + print desc + + +def check_feature_name(feature_name): + """ + Sanity-check the feature name for create/destroy. Return False if not OK. + """ + if not feature_name.endswith(".scc"): + print "Invalid feature name (must end with .scc) [%s], exiting" % feature_name + return False + + if "/" in feature_name: + print "Invalid feature name (don't specify directory) [%s], exiting" % feature_name + return False + + return True + + +def check_create_input(feature_items): + """ + Sanity-check the create input. Return False if not OK. + """ + if not check_feature_name(feature_items[0]): + return False + + if feature_items[1].endswith(".patch") or feature_items[1].startswith("CONFIG_"): + print "Missing description and/or compatibilty [%s], exiting" % feature_items[1] + return False + + if feature_items[2].endswith(".patch") or feature_items[2].startswith("CONFIG_"): + print "Missing description and/or compatibility [%s], exiting" % feature_items[1] + return False + + return True + + +def yocto_kernel_feature_create(scripts_path, machine, feature_items): + """ + Create a recipe-space kernel feature in a BSP. + """ + if not check_create_input(feature_items): + sys.exit(1) + + feature = feature_items[0] + feature_basename = feature.split(".")[0] + feature_description = feature_items[1] + feature_compat = feature_items[2] + + patches = [] + cfg_items = [] + + for item in feature_items[3:]: + if item.endswith(".patch"): + patches.append(item) + elif item.startswith("CONFIG"): + if ("=y" in item or "=m" in item): + cfg_items.append(item) + else: + print "Invalid feature item (must be .patch or CONFIG_*) [%s], exiting" % item + sys.exit(1) + + feature_dirname = "cfg" + if patches: + feature_dirname = "features" + + filesdir = find_filesdir(scripts_path, machine) + if not filesdir: + print "Couldn't add feature (%s), no 'files' dir found" % feature + sys.exit(1) + + featdir = os.path.join(filesdir, feature_dirname) + if not os.path.exists(featdir): + os.mkdir(featdir) + + for patch in patches: + if not os.path.isfile(patch): + print "Couldn't find patch (%s), exiting" % patch + sys.exit(1) + basename = os.path.basename(patch) + featdir_patch = os.path.join(featdir, basename) + shutil.copyfile(patch, featdir_patch) + + new_cfg_filename = os.path.join(featdir, feature_basename + ".cfg") + new_cfg_file = open(new_cfg_filename, "w") + for cfg_item in cfg_items: + new_cfg_file.write(cfg_item + "\n") + new_cfg_file.close() + + new_feature_filename = os.path.join(featdir, feature_basename + ".scc") + new_feature_file = open(new_feature_filename, "w") + new_feature_file.write("define KFEATURE_DESCRIPTION \"" + feature_description + "\"\n") + new_feature_file.write("define KFEATURE_COMPATIBILITY " + feature_compat + "\n\n") + + for patch in patches: + patch_dir, patch_file = os.path.split(patch) + new_feature_file.write("patch " + patch_file + "\n") + + new_feature_file.write("kconf non-hardware " + feature_basename + ".cfg\n") + new_feature_file.close() + + print "Added feature:" + print "\t%s" % feature_dirname + "/" + feature + + +def feature_in_use(scripts_path, machine, feature): + """ + Determine whether the specified feature is in use by the BSP. + Return True if so, False otherwise. + """ + features = read_features(scripts_path, machine) + for f in features: + if f == feature: + return True + return False + + +def feature_remove(scripts_path, machine, feature): + """ + Remove the specified feature from the available recipe-space + features defined for the BSP. + """ + features = read_features(scripts_path, machine) + new_features = [] + for f in features: + if f == feature: + continue + new_features.append(f) + write_features(scripts_path, machine, new_features) + + +def yocto_kernel_feature_destroy(scripts_path, machine, feature): + """ + Remove a recipe-space kernel feature from a BSP. + """ + if not check_feature_name(feature): + sys.exit(1) + + if feature_in_use(scripts_path, machine, "features/" + feature) or \ + feature_in_use(scripts_path, machine, "cfg/" + feature): + print "Feature %s is in use (use 'feature rm' to un-use it first), exiting" % feature + sys.exit(1) + + filesdir = find_filesdir(scripts_path, machine) + if not filesdir: + print "Couldn't destroy feature (%s), no 'files' dir found" % feature + sys.exit(1) + + feature_dirname = "features" + featdir = os.path.join(filesdir, feature_dirname) + if not os.path.exists(featdir): + print "Couldn't find feature directory (%s)" % feature_dirname + sys.exit(1) + + feature_fqn = os.path.join(featdir, feature) + if not os.path.exists(feature_fqn): + feature_dirname = "cfg" + featdir = os.path.join(filesdir, feature_dirname) + if not os.path.exists(featdir): + print "Couldn't find feature directory (%s)" % feature_dirname + sys.exit(1) + feature_fqn = os.path.join(featdir, feature_filename) + if not os.path.exists(feature_fqn): + print "Couldn't find feature (%s)" % feature + sys.exit(1) + + f = open(feature_fqn, "r") + lines = f.readlines() + for line in lines: + s = line.strip() + if s.startswith("patch ") or s.startswith("kconf "): + split_line = s.split() + filename = os.path.join(featdir, split_line[-1]) + if os.path.exists(filename): + os.remove(filename) + f.close() + os.remove(feature_fqn) + + feature_remove(scripts_path, machine, feature) + + print "Removed feature:" + print "\t%s" % feature_dirname + "/" + feature + + +def base_branches(context): + """ + Return a list of the base branches found in the kernel git repo. + """ + giturl = find_giturl(context) + + print "Getting branches from remote repo %s..." % giturl + + gitcmd = "git ls-remote %s *heads* 2>&1" % (giturl) + tmp = subprocess.Popen(gitcmd, shell=True, stdout=subprocess.PIPE).stdout.read() + + branches = [] + + if tmp: + tmpline = tmp.split("\n") + for line in tmpline: + if len(line)==0: + break; + if not line.endswith("base"): + continue; + idx = line.find("refs/heads/") + kbranch = line[idx + len("refs/heads/"):] + if kbranch.find("/") == -1 and kbranch.find("base") == -1: + continue + idx = kbranch.find("base") + branches.append(kbranch[:idx - 1]) + + return branches + + +def all_branches(context): + """ + Return a list of all the branches found in the kernel git repo. + """ + giturl = find_giturl(context) + + print "Getting branches from remote repo %s..." % giturl + + gitcmd = "git ls-remote %s *heads* 2>&1" % (giturl) + tmp = subprocess.Popen(gitcmd, shell=True, stdout=subprocess.PIPE).stdout.read() + + branches = [] + + base_prefixes = None + + try: + branches_base = context["branches_base"] + if branches_base: + base_prefixes = branches_base.split(":") + except KeyError: + pass + + arch = context["arch"] + + if tmp: + tmpline = tmp.split("\n") + for line in tmpline: + if len(line)==0: + break; + idx = line.find("refs/heads/") + kbranch = line[idx + len("refs/heads/"):] + kbranch_prefix = kbranch.rsplit("/", 1)[0] + + if base_prefixes: + for base_prefix in base_prefixes: + if kbranch_prefix == base_prefix: + branches.append(kbranch) + continue + + if (kbranch.find("/") != -1 and + (kbranch.find("standard") != -1 or kbranch.find("base") != -1) or + kbranch == "base"): + branches.append(kbranch) + continue + + return branches diff --git a/scripts/lib/bsp/substrate/target/arch/arm/.gitignore b/scripts/lib/bsp/substrate/target/arch/arm/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/.gitignore diff --git a/scripts/lib/bsp/substrate/target/arch/arm/conf/machine/machine.conf b/scripts/lib/bsp/substrate/target/arch/arm/conf/machine/machine.conf new file mode 100644 index 0000000..588367a --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/conf/machine/machine.conf @@ -0,0 +1,102 @@ +# yocto-bsp-filename {{=machine}}.conf +#@TYPE: Machine +#@NAME: {{=machine}} + +#@DESCRIPTION: Machine configuration for {{=machine}} systems + +{{ input type:"boolean" name:"xserver" prio:"50" msg:"Do you need support for X? (y/n)" default:"y" }} +{{ if xserver == "y": }} +PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg" +XSERVER ?= "xserver-xorg \ + xf86-input-evdev \ + xf86-input-mouse \ + xf86-video-fbdev \ + xf86-input-keyboard" + +MACHINE_EXTRA_RRECOMMENDS = " kernel-modules kernel-devicetree" + +EXTRA_IMAGEDEPENDS += "u-boot" + +{{ input type:"choicelist" name:"tunefile" prio:"40" msg:"Which machine tuning would you like to use?" default:"tune_cortexa8" }} +{{ input type:"choice" val:"tune_arm1136jf_s" msg:"arm1136jf-s tuning optimizations" }} +{{ input type:"choice" val:"tune_arm920t" msg:"arm920t tuning optimizations" }} +{{ input type:"choice" val:"tune_arm926ejs" msg:"arm926ejs tuning optimizations" }} +{{ input type:"choice" val:"tune_arm9tdmi" msg:"arm9tdmi tuning optimizations" }} +{{ input type:"choice" val:"tune_cortexa5" msg:"cortexa5 tuning optimizations" }} +{{ input type:"choice" val:"tune_cortexa7" msg:"cortexa7 tuning optimizations" }} +{{ input type:"choice" val:"tune_cortexa8" msg:"cortexa8 tuning optimizations" }} +{{ input type:"choice" val:"tune_cortexa9" msg:"cortexa9 tuning optimizations" }} +{{ input type:"choice" val:"tune_cortexa15" msg:"cortexa15 tuning optimizations" }} +{{ input type:"choice" val:"tune_cortexm1" msg:"cortexm1 tuning optimizations" }} +{{ input type:"choice" val:"tune_cortexm3" msg:"cortexm3 tuning optimizations" }} +{{ input type:"choice" val:"tune_cortexr4" msg:"cortexr4 tuning optimizations" }} +{{ input type:"choice" val:"tune_ep9312" msg:"ep9312 tuning optimizations" }} +{{ input type:"choice" val:"tune_iwmmxt" msg:"iwmmxt tuning optimizations" }} +{{ input type:"choice" val:"tune_strongarm1100" msg:"strongarm1100 tuning optimizations" }} +{{ input type:"choice" val:"tune_xscale" msg:"xscale tuning optimizations" }} +{{ if tunefile == "tune_arm1136jf_s": }} +include conf/machine/include/tune-arm1136jf-s.inc +{{ if tunefile == "tune_arm920t": }} +include conf/machine/include/tune-arm920t.inc +{{ if tunefile == "tune_arm926ejs": }} +include conf/machine/include/tune-arm926ejs.inc +{{ if tunefile == "tune_arm9tdmi": }} +include conf/machine/include/tune-arm9tdmi.inc +{{ if tunefile == "tune_cortexa5": }} +include conf/machine/include/tune-cortexa5.inc +{{ if tunefile == "tune_cortexa7": }} +include conf/machine/include/tune-cortexa7.inc +{{ if tunefile == "tune_cortexa8": }} +DEFAULTTUNE ?= "cortexa8hf-neon" +include conf/machine/include/tune-cortexa8.inc +{{ if tunefile == "tune_cortexa9": }} +include conf/machine/include/tune-cortexa9.inc +{{ if tunefile == "tune_cortexa15": }} +include conf/machine/include/tune-cortexa15.inc +{{ if tunefile == "tune_cortexm1": }} +include conf/machine/include/tune-cortexm1.inc +{{ if tunefile == "tune_cortexm3": }} +include conf/machine/include/tune-cortexm3.inc +{{ if tunefile == "tune_cortexr4": }} +include conf/machine/include/tune-cortexr4.inc +{{ if tunefile == "tune_ep9312": }} +include conf/machine/include/tune-ep9312.inc +{{ if tunefile == "tune_iwmmxt": }} +include conf/machine/include/tune-iwmmxt.inc +{{ if tunefile == "tune_strongarm1100": }} +include conf/machine/include/tune-strongarm1100.inc +{{ if tunefile == "tune_xscale": }} +include conf/machine/include/tune-xscale.inc + +IMAGE_FSTYPES += "tar.bz2 jffs2" +EXTRA_IMAGECMD_jffs2 = "-lnp " + +SERIAL_CONSOLE = "115200 ttyO0" + +{{ if kernel_choice == "custom": preferred_kernel = "linux-yocto-custom" }} +{{ if kernel_choice == "linux-yocto-dev": preferred_kernel = "linux-yocto-dev" }} +{{ if kernel_choice == "custom" or kernel_choice == "linux-yocto-dev" : }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" + +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel = kernel_choice.split('_')[0] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel_version = kernel_choice.split('_')[1] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" +PREFERRED_VERSION_{{=preferred_kernel}} ?= "{{=preferred_kernel_version}}%" + +KERNEL_IMAGETYPE = "uImage" +KERNEL_DEVICETREE = "am335x-bone.dtb am335x-boneblack.dtb" +KERNEL_EXTRA_ARGS += "LOADADDR=${UBOOT_ENTRYPOINT}" + +SPL_BINARY = "MLO" +UBOOT_SUFFIX = "img" +{{ input type:"edit" name:"uboot_machine" prio:"40" msg:"Please specify a value for UBOOT_MACHINE:" default:"am335x_evm_config" }} +UBOOT_MACHINE = "{{=uboot_machine}}" +{{ input type:"edit" name:"uboot_entrypoint" prio:"40" msg:"Please specify a value for UBOOT_ENTRYPOINT:" default:"0x80008000" }} +UBOOT_ENTRYPOINT = "{{=uboot_entrypoint}}" +{{ input type:"edit" name:"uboot_loadaddress" prio:"40" msg:"Please specify a value for UBOOT_LOADADDRESS:" default:"0x80008000" }} +UBOOT_LOADADDRESS = "{{=uboot_loadaddress}}" + +MACHINE_FEATURES = "usbgadget usbhost vfat alsa" + +IMAGE_BOOT_FILES ?= "u-boot.${UBOOT_SUFFIX} MLO" diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall b/scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall new file mode 100644 index 0000000..b442d02 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{=machine}} diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf b/scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf new file mode 100644 index 0000000..bc52893 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf @@ -0,0 +1,34 @@ +# yocto-bsp-filename {{ if xserver == "y": }} this +Section "Module" + Load "extmod" + Load "dbe" + Load "glx" + Load "freetype" + Load "type1" + Load "record" + Load "dri" +EndSection + +Section "Monitor" + Identifier "Builtin Default Monitor" +EndSection + +Section "Device" + Identifier "Builtin Default fbdev Device 0" + Driver "omapfb" +EndSection + +Section "Screen" + Identifier "Builtin Default fbdev Screen 0" + Device "Builtin Default fbdev Device 0" + Monitor "Builtin Default Monitor" +EndSection + +Section "ServerLayout" + Identifier "Builtin Default Layout" + Screen "Builtin Default fbdev Screen 0" +EndSection + +Section "ServerFlags" + Option "DontZap" "0" +EndSection diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend b/scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend new file mode 100644 index 0000000..3083003 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend @@ -0,0 +1,2 @@ +# yocto-bsp-filename {{ if xserver == "y": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:" diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files.noinstall b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files.noinstall new file mode 100644 index 0000000..1e0d92c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{ if kernel_choice != "custom": }} files diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-non_hardware.cfg b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-non_hardware.cfg new file mode 100644 index 0000000..9bfc90c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-non_hardware.cfg @@ -0,0 +1,31 @@ +# yocto-bsp-filename {{=machine}}-non_hardware.cfg +# +# Miscellaneous filesystems +# +CONFIG_NFS_DEF_FILE_IO_SIZE=1024 + +# +# Multiple Device Support +# +# CONFIG_MD is not set + +# Kernel Features +# +CONFIG_NO_HZ=y + +# +# CPUIdle +# +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_LADDER=y +CONFIG_CPU_IDLE_GOV_MENU=y + +# +# Kernel hacking +# +CONFIG_DEBUG_FS=y + +# +# Power management options +# +CONFIG_PM_DEBUG=y diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-preempt-rt.scc b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-preempt-rt.scc new file mode 100644 index 0000000..ea6966c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-preempt-rt.scc @@ -0,0 +1,15 @@ +# yocto-bsp-filename {{=machine}}-preempt-rt.scc +define KMACHINE {{=machine}} + +define KARCH arm + +include {{=map_preempt_rt_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc + +# default policy for preempt-rt kernels +include features/latencytop/latencytop.scc +include features/profiling/profiling.scc diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-standard.scc b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-standard.scc new file mode 100644 index 0000000..405972d --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-standard.scc @@ -0,0 +1,15 @@ +# yocto-bsp-filename {{=machine}}-standard.scc +define KMACHINE {{=machine}} + +define KARCH arm + +include {{=map_standard_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc + +# default policy for standard kernels +include features/latencytop/latencytop.scc +include features/profiling/profiling.scc diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-tiny.scc b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-tiny.scc new file mode 100644 index 0000000..921b7e7 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-tiny.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-tiny.scc +define KMACHINE {{=machine}} + +define KARCH arm + +include {{=map_tiny_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-config.cfg b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-config.cfg new file mode 100644 index 0000000..47489e4 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-config.cfg @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-config.cfg diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-features.scc b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-features.scc new file mode 100644 index 0000000..582759e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-features.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-features.scc diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-patches.scc b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-patches.scc new file mode 100644 index 0000000..97f747f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-patches.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine.cfg b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine.cfg new file mode 100644 index 0000000..a2e1ae0 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine.cfg @@ -0,0 +1,321 @@ +# yocto-bsp-filename {{=machine}}.cfg +# +# System Type +# +CONFIG_ARCH_OMAP=y + +# +# TI OMAP Implementations +# +# CONFIG_ARCH_OMAP2 is not set +CONFIG_ARCH_OMAP3=y + +# +# TI OMAP Common Features +# +CONFIG_ARCH_OMAP2PLUS=y + +# +# OMAP Feature Selections +# +CONFIG_OMAP_32K_TIMER=y +CONFIG_OMAP_32K_TIMER_HZ=128 +CONFIG_OMAP_DM_TIMER=y +CONFIG_OMAP_RESET_CLOCKS=y +CONFIG_OMAP_SMARTREFLEX=y +CONFIG_OMAP_SMARTREFLEX_CLASS3=y +CONFIG_OMAP_MBOX_FWK=m +CONFIG_OMAP_MBOX_KFIFO_SIZE=256 + +# +# OMAP Board Type +# +CONFIG_MACH_OMAP3_BEAGLE=y + +# +# Processor Features +# +CONFIG_ARM_THUMBEE=y +CONFIG_ARM_ERRATA_430973=y + +# +# Kernel Features +# +CONFIG_LEDS=y + + +# +# Serial drivers +# +CONFIG_SERIAL_OMAP=y +CONFIG_SERIAL_OMAP_CONSOLE=y + +# +# At least one emulation must be selected +# +CONFIG_VFP=y +CONFIG_NEON=y + +# +# Power management options +# +CONFIG_PM=y +CONFIG_PM_RUNTIME=y + +# +# Generic Driver Options +# +CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +# +# User Modules And Translation Layers +# +CONFIG_MTD_BLKDEVS=y +CONFIG_MTD_BLOCK=y + +# +# RAM/ROM/Flash chip drivers +# +CONFIG_MTD_CFI=y +CONFIG_MTD_CFI_INTELEXT=y + +# +# Disk-On-Chip Device Drivers +# +CONFIG_MTD_NAND=y + +CONFIG_MTD_NAND_OMAP2=y + +CONFIG_MTD_UBI=y + +# +# SCSI device support +# +CONFIG_SCSI=y + +# +# SCSI support type (disk, tape, CD-ROM) +# +CONFIG_BLK_DEV_SD=y + +# +# Ethernet (10 or 100Mbit) +# +CONFIG_SMSC911X=y +CONFIG_USB_NET_SMSC95XX=y + +# +# Userland interfaces +# +CONFIG_INPUT_EVDEV=y + +# +# Input Device Drivers +# +CONFIG_KEYBOARD_TWL4030=y +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_TOUCHSCREEN_ADS7846=y + +# +# Miscellaneous I2C Chip support +# +CONFIG_I2C=y +CONFIG_I2C_OMAP=y +CONFIG_SPI=y +CONFIG_SPI_MASTER=y +CONFIG_SPI_OMAP24XX=y + +# +# I2C GPIO expanders: +# +CONFIG_GPIO_TWL4030=y + +# +# SPI GPIO expanders: +# +CONFIG_OMAP_WATCHDOG=y +CONFIG_WATCHDOG_NOWAYOUT=y + +# +# Multifunction device drivers +# +CONFIG_TWL4030_CORE=y +CONFIG_REGULATOR=y +CONFIG_REGULATOR_DUMMY=y +CONFIG_REGULATOR_TWL4030=y + +# +# Graphics support +# +CONFIG_FB=y +CONFIG_DRM=m +# CONFIG_VGASTATE is not set +# CONFIG_VIDEO_OUTPUT_CONTROL is not set +# CONFIG_FIRMWARE_EDID is not set +# CONFIG_FB_DDC is not set +# CONFIG_FB_BOOT_VESA_SUPPORT is not set +CONFIG_FB_CFB_FILLRECT=y +CONFIG_FB_CFB_COPYAREA=y +CONFIG_FB_CFB_IMAGEBLIT=y +# CONFIG_FB_CFB_REV_PIXELS_IN_BYTE is not set +# CONFIG_FB_SYS_FILLRECT is not set +# CONFIG_FB_SYS_COPYAREA is not set +# CONFIG_FB_SYS_IMAGEBLIT is not set +# CONFIG_FB_FOREIGN_ENDIAN is not set +# CONFIG_FB_SYS_FOPS is not set +# CONFIG_FB_SVGALIB is not set +# CONFIG_FB_MACMODES is not set +# CONFIG_FB_BACKLIGHT is not set +CONFIG_FB_MODE_HELPERS=y +# CONFIG_FB_TILEBLITTING is not set + +# +# Frame buffer hardware drivers +# +# CONFIG_FB_S1D13XXX is not set +# CONFIG_FB_TMIO is not set +# CONFIG_FB_VIRTUAL is not set +# CONFIG_FB_METRONOME is not set +# CONFIG_FB_MB862XX is not set +# CONFIG_FB_BROADSHEET is not set +# CONFIG_FB_OMAP_BOOTLOADER_INIT is not set +CONFIG_OMAP2_VRAM=y +CONFIG_OMAP2_VRFB=y +CONFIG_OMAP2_DSS=y +CONFIG_OMAP2_VRAM_SIZE=14 +CONFIG_OMAP2_DSS_DEBUG_SUPPORT=y +# CONFIG_OMAP2_DSS_COLLECT_IRQ_STATS is not set +CONFIG_OMAP2_DSS_DPI=y +# CONFIG_OMAP2_DSS_RFBI is not set +CONFIG_OMAP2_DSS_VENC=y +# CONFIG_OMAP2_DSS_SDI is not set +CONFIG_OMAP2_DSS_DSI=y +# CONFIG_OMAP2_DSS_FAKE_VSYNC is not set +CONFIG_OMAP2_DSS_MIN_FCK_PER_PCK=0 +CONFIG_FB_OMAP2=y +CONFIG_FB_OMAP2_DEBUG_SUPPORT=y +CONFIG_FB_OMAP2_NUM_FBS=2 + +# +# OMAP2/3 Display Device Drivers +# +CONFIG_PANEL_GENERIC_DPI=y +CONFIG_PANEL_DVI=y +CONFIG_PANEL_SHARP_LS037V7DW01=y +# CONFIG_PANEL_LGPHILIPS_LB035Q02 is not set +# CONFIG_PANEL_TAAL is not set +CONFIG_PANEL_TPO_TD043MTEA1=m +# CONFIG_BACKLIGHT_LCD_SUPPORT is not set +CONFIG_BACKLIGHT_CLASS_DEVICE=y + +# +# Display device support +# +CONFIG_DISPLAY_SUPPORT=y +CONFIG_DUMMY_CONSOLE=y +# CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY is not set +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +# CONFIG_FONTS is not set +CONFIG_FONT_8x8=y +CONFIG_FONT_8x16=y +# CONFIG_LOGO_LINUX_MONO is not set +# CONFIG_LOGO_LINUX_VGA16 is not set + +# +# Console display driver support +# +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_LOGO=y +# CONFIG_VGA_CONSOLE is not set + +# DMA Devices +CONFIG_DMADEVICES=y +CONFIG_DMA_OMAP=y +CONFIG_DMA_OF=y + +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_SOC=y +CONFIG_SND_OMAP_SOC=y +CONFIG_SND_OMAP_SOC_OMAP_TWL4030=y + +# +# USB Input Devices +# +CONFIG_USB=y +CONFIG_USB_SUPPORT=y + +# +# Miscellaneous USB options +# +CONFIG_USB_OTG=y +# CONFIG_USB_OTG_WHITELIST is not set + +# +# USB Host Controller Drivers +# +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_TT_NEWSCHED=y +CONFIG_USB_EHCI_ROOT_HUB_TT=y +CONFIG_USB_MUSB_HDRC=y +CONFIG_USB_MUSB_OMAP2PLUS=y +CONFIG_USB_OMAP=y + +# +# OMAP 343x high speed USB support +# +CONFIG_USB_MUSB_OTG=y +CONFIG_USB_GADGET_MUSB_HDRC=y +CONFIG_USB_MUSB_HDRC_HCD=y +CONFIG_USB_INVENTRA_DMA=y + +# +# NOTE: USB_STORAGE enables SCSI, and 'SCSI disk support' +# + +# +# may also be needed; see USB_STORAGE Help for more information +# +CONFIG_USB_STORAGE=y + +# +# USB Miscellaneous drivers +# +CONFIG_USB_GADGET=y +CONFIG_USB_GADGET_DUALSPEED=y +CONFIG_USB_OTG_UTILS=y +CONFIG_TWL4030_USB=y + +# USB gadget modules +CONFIG_USB_G_NCM=y +CONFIG_USB_MASS_STORAGE=y + +CONFIG_MMC=y + +# +# MMC/SD Host Controller Drivers +# +CONFIG_MMC_OMAP_HS=y + +# +# Real Time Clock +# +CONFIG_RTC_LIB=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_TWL4030=y + +# +# DOS/FAT/NT Filesystems +# +CONFIG_VFAT_FS=y + +# +# Multimedia core support +# + +# CONFIG_VIDEO_HELPER_CHIPS_AUTO is not set + +# +# Advanced Power Management Emulation support +# +CONFIG_APM_EMULATION=y diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine.scc b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine.scc new file mode 100644 index 0000000..828400d --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine.scc @@ -0,0 +1,8 @@ +# yocto-bsp-filename {{=machine}}.scc +kconf hardware {{=machine}}.cfg +kconf non-hardware {{machine}}-non_hardware.cfg + +include features/usb-net/usb-net.scc + +kconf hardware {{=machine}}-user-config.cfg +include {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/kernel-list.noinstall b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/kernel-list.noinstall new file mode 100644 index 0000000..00cf360 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/kernel-list.noinstall @@ -0,0 +1,5 @@ +{{ if kernel_choice != "custom": }} +{{ input type:"boolean" name:"use_default_kernel" prio:"10" msg:"Would you like to use the default (4.4) kernel? (y/n)" default:"y"}} + +{{ if kernel_choice != "custom" and use_default_kernel == "n": }} +{{ input type:"choicelist" name:"kernel_choice" gen:"bsp.kernel.kernels" prio:"10" msg:"Please choose the kernel to use in this BSP:" default:"linux-yocto_4.4"}} diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-dev.bbappend b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-dev.bbappend new file mode 100644 index 0000000..c336007 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-dev.bbappend @@ -0,0 +1,25 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-dev": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Would you like SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend new file mode 100644 index 0000000..0a47a4e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend new file mode 100644 index 0000000..815c77b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto_4.1.bbappend new file mode 100644 index 0000000..2d3d073 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto_4.1.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1"
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto_4.4.bbappend new file mode 100644 index 0000000..b88a06c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto_4.4.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/common/COPYING.MIT b/scripts/lib/bsp/substrate/target/arch/common/COPYING.MIT new file mode 100644 index 0000000..fb950dc --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/COPYING.MIT @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/scripts/lib/bsp/substrate/target/arch/common/README b/scripts/lib/bsp/substrate/target/arch/common/README new file mode 100644 index 0000000..928659f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/README @@ -0,0 +1,118 @@ +This README file contains information on building the meta-{{=machine}} +BSP layer, and booting the images contained in the /binary directory. +Please see the corresponding sections below for details. + + +Dependencies +============ + +This layer depends on: + + URI: git://git.openembedded.org/bitbake + branch: master + + URI: git://git.openembedded.org/openembedded-core + layers: meta + branch: master + + URI: git://git.yoctoproject.org/xxxx + layers: xxxx + branch: master + + +Patches +======= + +Please submit any patches against this BSP to the Yocto mailing list +(yocto@yoctoproject.org) and cc: the maintainer: + +Maintainer: XXX YYYYYY <xxx.yyyyyy@zzzzz.com> + +Please see the meta-xxxx/MAINTAINERS file for more details. + + +Table of Contents +================= + + I. Building the meta-{{=machine}} BSP layer + II. Booting the images in /binary + + +I. Building the meta-{{=machine}} BSP layer +======================================== + +--- replace with specific instructions for your layer --- + +In order to build an image with BSP support for a given release, you +need to download the corresponding BSP tarball from the 'Board Support +Package (BSP) Downloads' page of the Yocto Project website. + +Having done that, and assuming you extracted the BSP tarball contents +at the top-level of your yocto build tree, you can build a +{{=machine}} image by adding the location of the meta-{{=machine}} +layer to bblayers.conf, along with any other layers needed (to access +common metadata shared between BSPs) e.g.: + + yocto/meta-xxxx \ + yocto/meta-xxxx/meta-{{=machine}} \ + +To enable the {{=machine}} layer, add the {{=machine}} MACHINE to local.conf: + + MACHINE ?= "{{=machine}}" + +You should then be able to build a {{=machine}} image as such: + + $ source oe-init-build-env + $ bitbake core-image-sato + +At the end of a successful build, you should have a live image that +you can boot from a USB flash drive (see instructions on how to do +that below, in the section 'Booting the images from /binary'). + +As an alternative to downloading the BSP tarball, you can also work +directly from the meta-xxxx git repository. For each BSP in the +'meta-xxxx' repository, there are multiple branches, one corresponding +to each major release starting with 'laverne' (0.90), in addition to +the latest code which tracks the current master (note that not all +BSPs are present in every release). Instead of extracting a BSP +tarball at the top level of your yocto build tree, you can +equivalently check out the appropriate branch from the meta-xxxx +repository at the same location. + + +II. Booting the images in /binary +================================= + +--- replace with specific instructions for your platform --- + +This BSP contains bootable live images, which can be used to directly +boot Yocto off of a USB flash drive. + +Under Linux, insert a USB flash drive. Assuming the USB flash drive +takes device /dev/sdf, use dd to copy the live image to it. For +example: + +# dd if=core-image-sato-{{=machine}}-20101207053738.hddimg of=/dev/sdf +# sync +# eject /dev/sdf + +This should give you a bootable USB flash device. Insert the device +into a bootable USB socket on the target, and power on. This should +result in a system booted to the Sato graphical desktop. + +If you want a terminal, use the arrows at the top of the UI to move to +different pages of available applications, one of which is named +'Terminal'. Clicking that should give you a root terminal. + +If you want to ssh into the system, you can use the root terminal to +ifconfig the IP address and use that to ssh in. The root password is +empty, so to log in type 'root' for the user name and hit 'Enter' at +the Password prompt: and you should be in. + +---- + +If you find you're getting corrupt images on the USB (it doesn't show +the syslinux boot: prompt, or the boot: prompt contains strange +characters), try doing this first: + +# dd if=/dev/zero of=/dev/sdf bs=1M count=512 diff --git a/scripts/lib/bsp/substrate/target/arch/common/README.sources b/scripts/lib/bsp/substrate/target/arch/common/README.sources new file mode 100644 index 0000000..3c4cb7b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/README.sources @@ -0,0 +1,17 @@ +The sources for the packages comprising the images shipped with this +BSP can be found at the following location: + +http://downloads.yoctoproject.org/mirror/sources/ + +The metadata used to generate the images shipped with this BSP, in +addition to the code contained in this BSP, can be found at the +following location: + +http://www.yoctoproject.org/downloads/yocto-1.1/poky-edison-6.0.tar.bz2 + +The metadata used to generate the images shipped with this BSP, in +addition to the code contained in this BSP, can also be found at the +following locations: + +git://git.yoctoproject.org/poky.git +git://git.yoctoproject.org/meta-xxxx diff --git a/scripts/lib/bsp/substrate/target/arch/common/binary/.gitignore b/scripts/lib/bsp/substrate/target/arch/common/binary/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/binary/.gitignore diff --git a/scripts/lib/bsp/substrate/target/arch/common/conf/layer.conf b/scripts/lib/bsp/substrate/target/arch/common/conf/layer.conf new file mode 100644 index 0000000..5529f45 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/conf/layer.conf @@ -0,0 +1,10 @@ +# We have a conf and classes directory, add to BBPATH +BBPATH .= ":${LAYERDIR}" + +# We have a recipes-* directories, add to BBFILES +BBFILES += "${LAYERDIR}/recipes-*/*/*.bb \ + ${LAYERDIR}/recipes-*/*/*.bbappend" + +BBFILE_COLLECTIONS += "{{=machine}}" +BBFILE_PATTERN_{{=machine}} = "^${LAYERDIR}/" +BBFILE_PRIORITY_{{=machine}} = "6" diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor/machine.noinstall b/scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor/machine.noinstall new file mode 100644 index 0000000..b442d02 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor/machine.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{=machine}} diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor/machine/machconfig b/scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor/machine/machconfig new file mode 100644 index 0000000..3b85d38 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor/machine/machconfig @@ -0,0 +1,5 @@ +# Assume a USB mouse and keyboard are connected +{{ input type:"boolean" name:"touchscreen" msg:"Does your BSP have a touchscreen? (y/n)" default:"n" }} +HAVE_TOUCHSCREEN={{=touchscreen}} +{{ input type:"boolean" name:"keyboard" msg:"Does your BSP have a keyboard? (y/n)" default:"y" }} +HAVE_KEYBOARD={{=keyboard}} diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor_0.0.bbappend b/scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor_0.0.bbappend new file mode 100644 index 0000000..6d4804d --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor_0.0.bbappend @@ -0,0 +1,2 @@ +FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:" + diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/kernel-list.noinstall b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/kernel-list.noinstall new file mode 100644 index 0000000..663dddb --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/kernel-list.noinstall @@ -0,0 +1,26 @@ +{{ if kernel_choice == "custom": }} +{{ input type:"boolean" name:"custom_kernel_remote" prio:"20" msg:"Is the custom kernel you'd like to use in a remote git repo? (y/n)" default:"y"}} + +{{ if kernel_choice == "custom" and custom_kernel_remote == "y": }} +{{ input type:"edit-git-repo" name:"custom_kernel_remote_path" prio:"20" msg:"Please enter the full URI to the remote git repo (the default corresponds to linux-stable v3.16.3)" default:"git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git"}} + +{{ if kernel_choice == "custom" and custom_kernel_remote == "n": }} +{{ input type:"edit-git-repo" name:"custom_kernel_local_path" prio:"20" msg:"You've indicated that you're not using a remote git repo. Please enter the full path to the local git repo you want to use (the default assumes a local linux-stable v3.16.3)" default:"/home/trz/yocto/kernels/linux-stable.git"}} + +{{ if kernel_choice == "custom": }} +{{ input type:"boolean" name:"custom_kernel_need_kbranch" prio:"20" msg:"Do you need to use a specific (non-master) branch? (y/n)" default:"n"}} + +{{ if kernel_choice == "custom" and custom_kernel_need_kbranch == "y": }} +{{ input type:"edit" name:"custom_kernel_kbranch" prio:"20" msg:"Please enter the branch you want to use (the default branch corresponds to the linux-stable 'linux-3.16.y' branch):" default:"linux-3.16.y"}} + +{{ if kernel_choice == "custom": }} +{{ input type:"edit" name:"custom_kernel_srcrev" prio:"20" msg:"Please enter the SRCREV (commit id) you'd like to use (use '${AUTOREV}' to track the current HEAD):" default:"${AUTOREV}"}} + +{{ if kernel_choice == "custom": }} +{{ input type:"edit" name:"custom_kernel_linux_version" prio:"20" msg:"Please enter the Linux version of the kernel you've specified:" default:"3.16.3"}} + +{{ if kernel_choice == "custom": }} +{{ input type:"edit" name:"custom_kernel_linux_version_extension" prio:"20" msg:"Please enter a Linux version extension if you want (it will show up at the end of the kernel name shown by uname):" default:"-custom"}} + +{{ if kernel_choice == "custom": }} +{{ input type:"edit-file" name:"custom_kernel_defconfig" prio:"20" msg:"It's recommended (but not required) that custom kernels be built using a defconfig. Please enter the full path to the defconfig for your kernel (NOTE: if you don't specify a defconfig the kernel probably won't build or boot):" default:""}} diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom.bb b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom.bb new file mode 100644 index 0000000..fda955b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom.bb @@ -0,0 +1,57 @@ +# yocto-bsp-filename {{ if kernel_choice == "custom": }} this +# This file was derived from the linux-yocto-custom.bb recipe in +# oe-core. +# +# linux-yocto-custom.bb: +# +# A yocto-bsp-generated kernel recipe that uses the linux-yocto and +# oe-core kernel classes to apply a subset of yocto kernel +# management to git managed kernel repositories. +# +# Warning: +# +# Building this kernel without providing a defconfig or BSP +# configuration will result in build or boot errors. This is not a +# bug. +# +# Notes: +# +# patches: patches can be merged into to the source git tree itself, +# added via the SRC_URI, or controlled via a BSP +# configuration. +# +# example configuration addition: +# SRC_URI += "file://smp.cfg" +# example patch addition: +# SRC_URI += "file://0001-linux-version-tweak.patch +# example feature addition: +# SRC_URI += "file://feature.scc" +# + +inherit kernel +require recipes-kernel/linux/linux-yocto.inc + +{{ if kernel_choice == "custom" and custom_kernel_remote == "y": }} +SRC_URI = "{{=custom_kernel_remote_path}};protocol=git;bareclone=1;branch=${KBRANCH}" +{{ if kernel_choice == "custom" and custom_kernel_remote == "n": }} +SRC_URI = "git://{{=custom_kernel_local_path}};protocol=file;bareclone=1;branch=${KBRANCH}" + +SRC_URI += "file://defconfig" + +SRC_URI += "file://{{=machine}}.scc \ + file://{{=machine}}.cfg \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + " + +{{ if kernel_choice == "custom" and custom_kernel_need_kbranch == "y" and custom_kernel_kbranch and custom_kernel_kbranch != "master": }} +KBRANCH = "{{=custom_kernel_kbranch}}" + +LINUX_VERSION ?= "{{=custom_kernel_linux_version}}" +LINUX_VERSION_EXTENSION ?= "{{=custom_kernel_linux_version_extension}}" + +SRCREV="{{=custom_kernel_srcrev}}" + +PV = "${LINUX_VERSION}+git${SRCPV}" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom.noinstall b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom.noinstall new file mode 100644 index 0000000..017d206 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{ if kernel_choice == "custom": }} linux-yocto-custom diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/defconfig b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/defconfig new file mode 100644 index 0000000..ceb0ffa --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/defconfig @@ -0,0 +1,5 @@ +# +# Placeholder for custom default kernel configuration. yocto-bsp will +# replace this file with a user-specified defconfig. +# +{{ if custom_kernel_defconfig: replace_file(of, custom_kernel_defconfig) }} diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine-user-config.cfg b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine-user-config.cfg new file mode 100644 index 0000000..922309d --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine-user-config.cfg @@ -0,0 +1,9 @@ +# yocto-bsp-filename {{=machine}}-user-config.cfg +# +# Used by yocto-kernel to manage config options. +# +# yocto-kernel may change the contents of this file in any +# way it sees fit, including removing comments like this, +# so don't manually make any modifications you don't want +# to lose. +# diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine-user-patches.scc b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine-user-patches.scc new file mode 100644 index 0000000..6d1138f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine-user-patches.scc @@ -0,0 +1,9 @@ +# yocto-bsp-filename {{=machine}}-user-patches.scc +# +# Used by yocto-kernel to manage patches. +# +# yocto-kernel may change the contents of this file in any +# way it sees fit, including removing comments like this, +# so don't manually make any modifications you don't want +# to lose. +# diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine.cfg b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine.cfg new file mode 100644 index 0000000..1ba8201 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine.cfg @@ -0,0 +1,4 @@ +# yocto-bsp-filename {{=machine}}.cfg +# +# A convenient place to add config options, nothing more. +# diff --git a/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine.scc b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine.scc new file mode 100644 index 0000000..0b6b413 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine.scc @@ -0,0 +1,18 @@ +# yocto-bsp-filename {{=machine}}.scc +# +# The top-level 'feature' for the {{=machine}} custom kernel. +# +# Essentially this is a convenient top-level container or starting +# point for adding lower-level config fragements and features. +# + +# {{=machine}}.cfg in the linux-yocto-custom subdir is just a +# convenient place for adding random config fragments. + +kconf hardware {{=machine}}.cfg + +# These are used by yocto-kernel to add config fragments and features. +# Don't remove if you plan on using yocto-kernel with this BSP. + +kconf hardware {{=machine}}-user-config.cfg +include {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/i386/conf/machine/machine.conf b/scripts/lib/bsp/substrate/target/arch/i386/conf/machine/machine.conf new file mode 100644 index 0000000..d5abe4f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/conf/machine/machine.conf @@ -0,0 +1,77 @@ +# yocto-bsp-filename {{=machine}}.conf +#@TYPE: Machine +#@NAME: {{=machine}} + +#@DESCRIPTION: Machine configuration for {{=machine}} systems + +{{ if kernel_choice == "custom": preferred_kernel = "linux-yocto-custom" }} +{{ if kernel_choice == "linux-yocto-dev": preferred_kernel = "linux-yocto-dev" }} +{{ if kernel_choice == "custom" or kernel_choice == "linux-yocto-dev" : }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" + +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel = kernel_choice.split('_')[0] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel_version = kernel_choice.split('_')[1] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" +PREFERRED_VERSION_{{=preferred_kernel}} ?= "{{=preferred_kernel_version}}%" + +{{ input type:"choicelist" name:"tunefile" prio:"40" msg:"Which machine tuning would you like to use?" default:"tune_core2" }} +{{ input type:"choice" val:"tune_i586" msg:"i586 tuning optimizations" }} +{{ input type:"choice" val:"tune_atom" msg:"Atom tuning optimizations" }} +{{ input type:"choice" val:"tune_core2" msg:"Core2 tuning optimizations" }} +{{ if tunefile == "tune_i586": }} +require conf/machine/include/tune-i586.inc +{{ if tunefile == "tune_atom": }} +require conf/machine/include/tune-atom.inc +{{ if tunefile == "tune_core2": }} +DEFAULTTUNE="core2-32" +require conf/machine/include/tune-core2.inc + +require conf/machine/include/x86-base.inc + +MACHINE_FEATURES += "wifi efi pcbios" + +{{ input type:"boolean" name:"xserver" prio:"50" msg:"Do you need support for X? (y/n)" default:"y" }} + +{{ if xserver == "y" and (kernel_choice == "linux-yocto_4.4" or kernel_choice == "linux-yocto_4.1"): }} +{{ input type:"choicelist" name:"xserver_choice" prio:"50" msg:"Please select an xserver for this machine:" default:"xserver_vesa" }} +{{ input type:"choice" val:"xserver_vesa" msg:"VESA xserver support" }} +{{ input type:"choice" val:"xserver_i915" msg:"i915 xserver support" }} +{{ input type:"choice" val:"xserver_i965" msg:"i965 xserver support" }} +{{ input type:"choice" val:"xserver_fbdev" msg:"fbdev xserver support" }} +{{ input type:"choice" val:"xserver_modesetting" msg:"modesetting xserver support" }} + +{{ if xserver == "y" and kernel_choice == "custom": }} +{{ input type:"choicelist" name:"xserver_choice" prio:"50" msg:"Please select an xserver for this machine:" default:"xserver_vesa" }} +{{ input type:"choice" val:"xserver_vesa" msg:"VESA xserver support" }} +{{ input type:"choice" val:"xserver_i915" msg:"i915 xserver support" }} +{{ input type:"choice" val:"xserver_i965" msg:"i965 xserver support" }} +{{ input type:"choice" val:"xserver_fbdev" msg:"fbdev xserver support" }} +{{ input type:"choice" val:"xserver_modesetting" msg:"modesetting xserver support" }} + +{{ if xserver == "y" and kernel_choice != "linux-yocto_4.4" and kernel_choice != "linux-yocto_4.1" and kernel_choice != "custom": xserver_choice = "xserver_i915" }} + +{{ if xserver == "y": }} +XSERVER ?= "${XSERVER_X86_BASE} \ + ${XSERVER_X86_EXT} \ +{{ if xserver == "y" and xserver_choice == "xserver_vesa": }} + ${XSERVER_X86_VESA} \ +{{ if xserver == "y" and xserver_choice == "xserver_i915": }} + ${XSERVER_X86_I915} \ +{{ if xserver == "y" and xserver_choice == "xserver_i965": }} + ${XSERVER_X86_I965} \ +{{ if xserver == "y" and xserver_choice == "xserver_fbdev": }} + ${XSERVER_X86_FBDEV} \ +{{ if xserver == "y" and xserver_choice == "xserver_modesetting": }} + ${XSERVER_X86_MODESETTING} \ +{{ if xserver == "y": }} + " + +MACHINE_EXTRA_RRECOMMENDS += "linux-firmware v86d eee-acpi-scripts" + +EXTRA_OECONF_append_pn-matchbox-panel-2 = " --with-battery=acpi" + +GLIBC_ADDONS = "nptl" + +{{ if xserver == "y" and xserver_choice == "xserver_vesa": }} +APPEND += "video=vesafb vga=0x318" diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall b/scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall new file mode 100644 index 0000000..b442d02 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{=machine}} diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf b/scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf new file mode 100644 index 0000000..ac9a0f1 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf @@ -0,0 +1 @@ +# yocto-bsp-filename {{ if xserver == "y": }} this diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend b/scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend new file mode 100644 index 0000000..3083003 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend @@ -0,0 +1,2 @@ +# yocto-bsp-filename {{ if xserver == "y": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:" diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files.noinstall b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files.noinstall new file mode 100644 index 0000000..1e0d92c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{ if kernel_choice != "custom": }} files diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-preempt-rt.scc b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-preempt-rt.scc new file mode 100644 index 0000000..7146e23 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-preempt-rt.scc @@ -0,0 +1,17 @@ +# yocto-bsp-filename {{=machine}}-preempt-rt.scc +define KMACHINE {{=machine}} + +define KARCH i386 + +include {{=map_preempt_rt_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc + +# default policy for preempt-rt kernels +include cfg/usb-mass-storage.scc +include cfg/boot-live.scc +include features/latencytop/latencytop.scc +include features/profiling/profiling.scc diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-standard.scc b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-standard.scc new file mode 100644 index 0000000..67a54be --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-standard.scc @@ -0,0 +1,17 @@ +# yocto-bsp-filename {{=machine}}-standard.scc +define KMACHINE {{=machine}} + +define KARCH i386 + +include {{=map_standard_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc + +# default policy for standard kernels +include cfg/usb-mass-storage.scc +include cfg/boot-live.scc +include features/latencytop/latencytop.scc +include features/profiling/profiling.scc diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-tiny.scc b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-tiny.scc new file mode 100644 index 0000000..91373b3 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-tiny.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-tiny.scc +define KMACHINE {{=machine}} + +define KARCH i386 + +include {{=map_tiny_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-config.cfg b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-config.cfg new file mode 100644 index 0000000..69efdcc --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-config.cfg @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-config.cfg
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-features.scc b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-features.scc new file mode 100644 index 0000000..85be26d --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-features.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-features.scc
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-patches.scc b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-patches.scc new file mode 100644 index 0000000..4c59daa --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-patches.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-patches.scc
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine.cfg b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine.cfg new file mode 100644 index 0000000..3b168b7 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine.cfg @@ -0,0 +1,55 @@ +# yocto-bsp-filename {{=machine}}.cfg +CONFIG_X86_32=y +CONFIG_MATOM=y +CONFIG_PRINTK=y + +# Basic hardware support for the box - network, USB, PCI, sound +CONFIG_NETDEVICES=y +CONFIG_ATA=y +CONFIG_ATA_GENERIC=y +CONFIG_ATA_SFF=y +CONFIG_PCI=y +CONFIG_MMC=y +CONFIG_MMC_SDHCI=y +CONFIG_USB_SUPPORT=y +CONFIG_USB=y +CONFIG_USB_ARCH_HAS_EHCI=y +CONFIG_R8169=y +CONFIG_PATA_SCH=y +CONFIG_MMC_SDHCI_PCI=y +CONFIG_USB_EHCI_HCD=y +CONFIG_PCIEPORTBUS=y +CONFIG_NET=y +CONFIG_USB_UHCI_HCD=y +CONFIG_USB_OHCI_HCD=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_SG=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_HDA_INTEL=y +CONFIG_SATA_AHCI=y +CONFIG_AGP=y +CONFIG_PM=y +CONFIG_ACPI=y +CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_INPUT=y + +# Make sure these are on, otherwise the bootup won't be fun +CONFIG_EXT3_FS=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_MODULES=y +CONFIG_SHMEM=y +CONFIG_TMPFS=y +CONFIG_PACKET=y + +# Needed for booting (and using) USB memory sticks +CONFIG_BLK_DEV_LOOP=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y + +CONFIG_RD_GZIP=y + +# Needed for booting (and using) CD images +CONFIG_BLK_DEV_SR=y diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine.scc b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine.scc new file mode 100644 index 0000000..3d32f11 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine.scc @@ -0,0 +1,21 @@ +# yocto-bsp-filename {{=machine}}.scc +kconf hardware {{=machine}}.cfg + +include features/intel-e1xxxx/intel-e100.scc +include features/intel-e1xxxx/intel-e1xxxx.scc + +{{ if xserver == "y" and xserver_choice == "xserver_i915" or xserver_choice == "xserver_i965": }} +include features/i915/i915.scc + +include features/serial/8250.scc +include features/ericsson-3g/f5521gw.scc + +{{ if xserver == "y" and xserver_choice == "xserver_vesa": }} +include cfg/vesafb.scc + +include cfg/usb-mass-storage.scc +include cfg/boot-live.scc +include features/power/intel.scc + +kconf hardware {{=machine}}-user-config.cfg +include {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/kernel-list.noinstall b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/kernel-list.noinstall new file mode 100644 index 0000000..00cf360 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/kernel-list.noinstall @@ -0,0 +1,5 @@ +{{ if kernel_choice != "custom": }} +{{ input type:"boolean" name:"use_default_kernel" prio:"10" msg:"Would you like to use the default (4.4) kernel? (y/n)" default:"y"}} + +{{ if kernel_choice != "custom" and use_default_kernel == "n": }} +{{ input type:"choicelist" name:"kernel_choice" gen:"bsp.kernel.kernels" prio:"10" msg:"Please choose the kernel to use in this BSP:" default:"linux-yocto_4.4"}} diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-dev.bbappend b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-dev.bbappend new file mode 100644 index 0000000..c336007 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-dev.bbappend @@ -0,0 +1,25 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-dev": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Would you like SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend new file mode 100644 index 0000000..0a47a4e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend new file mode 100644 index 0000000..815c77b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto_4.1.bbappend new file mode 100644 index 0000000..aecdff0 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto_4.1.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard:standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard:standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto_4.4.bbappend new file mode 100644 index 0000000..dd4de31 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto_4.4.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard:standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard:standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/layer/COPYING.MIT b/scripts/lib/bsp/substrate/target/arch/layer/COPYING.MIT new file mode 100644 index 0000000..89de354 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/COPYING.MIT @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/scripts/lib/bsp/substrate/target/arch/layer/README b/scripts/lib/bsp/substrate/target/arch/layer/README new file mode 100644 index 0000000..ca6527c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/README @@ -0,0 +1,64 @@ +This README file contains information on the contents of the +{{=layer_name}} layer. + +Please see the corresponding sections below for details. + + +Dependencies +============ + +This layer depends on: + + URI: git://git.openembedded.org/bitbake + branch: master + + URI: git://git.openembedded.org/openembedded-core + layers: meta + branch: master + + URI: git://git.yoctoproject.org/xxxx + layers: xxxx + branch: master + + +Patches +======= + +Please submit any patches against the {{=layer_name}} layer to the +xxxx mailing list (xxxx@zzzz.org) and cc: the maintainer: + +Maintainer: XXX YYYYYY <xxx.yyyyyy@zzzzz.com> + + +Table of Contents +================= + + I. Adding the {{=layer_name}} layer to your build + II. Misc + + +I. Adding the {{=layer_name}} layer to your build +================================================= + +--- replace with specific instructions for the {{=layer_name}} layer --- + +In order to use this layer, you need to make the build system aware of +it. + +Assuming the {{=layer_name}} layer exists at the top-level of your +yocto build tree, you can add it to the build system by adding the +location of the {{=layer_name}} layer to bblayers.conf, along with any +other layers needed. e.g.: + + BBLAYERS ?= " \ + /path/to/yocto/meta \ + /path/to/yocto/meta-poky \ + /path/to/yocto/meta-yocto-bsp \ + /path/to/yocto/meta-{{=layer_name}} \ + " + + +II. Misc +======== + +--- replace with specific information about the {{=layer_name}} layer --- diff --git a/scripts/lib/bsp/substrate/target/arch/layer/conf/layer.conf b/scripts/lib/bsp/substrate/target/arch/layer/conf/layer.conf new file mode 100644 index 0000000..bdffe17 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/conf/layer.conf @@ -0,0 +1,10 @@ +# We have a conf and classes directory, add to BBPATH +BBPATH .= ":${LAYERDIR}" + +# We have recipes-* directories, add to BBFILES +BBFILES += "${LAYERDIR}/recipes-*/*/*.bb \ + ${LAYERDIR}/recipes-*/*/*.bbappend" + +BBFILE_COLLECTIONS += "{{=layer_name}}" +BBFILE_PATTERN_{{=layer_name}} = "^${LAYERDIR}/" +BBFILE_PRIORITY_{{=layer_name}} = "{{=layer_priority}}" diff --git a/scripts/lib/bsp/substrate/target/arch/layer/layer-questions.noinstall b/scripts/lib/bsp/substrate/target/arch/layer/layer-questions.noinstall new file mode 100644 index 0000000..e2a89c3 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/layer-questions.noinstall @@ -0,0 +1,14 @@ +{{ input type:"edit" name:"layer_priority" prio:"20" msg:"Please enter the layer priority you'd like to use for the layer:" default:"6"}} + +{{ input type:"boolean" name:"create_example_recipe" prio:"20" msg:"Would you like to have an example recipe created? (y/n)" default:"n"}} + +{{ if create_example_recipe == "y": }} +{{ input type:"edit" name:"example_recipe_name" prio:"20" msg:"Please enter the name you'd like to use for your example recipe:" default:"example"}} + +{{ input type:"boolean" name:"create_example_bbappend" prio:"20" msg:"Would you like to have an example bbappend file created? (y/n)" default:"n"}} + +{{ if create_example_bbappend == "y": }} +{{ input type:"edit" name:"example_bbappend_name" prio:"20" msg:"Please enter the name you'd like to use for your bbappend file:" default:"example"}} + +{{ if create_example_bbappend == "y": }} +{{ input type:"edit" name:"example_bbappend_version" prio:"20" msg:"Please enter the version number you'd like to use for your bbappend file (this should match the recipe you're appending to):" default:"0.1"}} diff --git a/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend.noinstall b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend.noinstall new file mode 100644 index 0000000..3594e65 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{ if create_example_bbappend == "y": }} recipes-example-bbappend diff --git a/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version.bbappend b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version.bbappend new file mode 100644 index 0000000..3531330 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version.bbappend @@ -0,0 +1,9 @@ +# yocto-bsp-filename {{=example_bbappend_name}}_{{=example_bbappend_version}}.bbappend +FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}-${PV}:" + +# +# This .bbappend doesn't yet do anything - replace this text with +# modifications to the example_0.1.bb recipe, or whatever recipe it is +# that you want to modify with this .bbappend (make sure you change +# the recipe name (PN) and version (PV) to match). +# diff --git a/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version.noinstall b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version.noinstall new file mode 100644 index 0000000..46df8a8 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{=example_bbappend_name}}-{{=example_bbappend_version}} diff --git a/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version/example.patch b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version/example.patch new file mode 100644 index 0000000..2000a34 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version/example.patch @@ -0,0 +1,12 @@ +# +# This is a non-functional placeholder file, here for example purposes +# only. +# +# If you had a patch for your recipe, you'd put it in this directory +# and reference it from your recipe's SRC_URI: +# +# SRC_URI += "file://example.patch" +# +# Note that you could also rename the directory containing this patch +# to remove the version number or simply rename it 'files'. Doing so +# allows you to use the same directory for multiple recipes. diff --git a/scripts/lib/bsp/substrate/target/arch/layer/recipes-example.noinstall b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example.noinstall new file mode 100644 index 0000000..b0069b1 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{ if create_example_recipe == "y": }} recipes-example diff --git a/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1.bb b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1.bb new file mode 100644 index 0000000..5fbf594 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1.bb @@ -0,0 +1,23 @@ +# yocto-bsp-filename {{=example_recipe_name}}_0.1.bb +# +# This file was derived from the 'Hello World!' example recipe in the +# Yocto Project Development Manual. +# + +SUMMARY = "Simple helloworld application" +SECTION = "examples" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +SRC_URI = "file://helloworld.c" + +S = "${WORKDIR}" + +do_compile() { + ${CC} helloworld.c -o helloworld +} + +do_install() { + install -d ${D}${bindir} + install -m 0755 helloworld ${D}${bindir} +} diff --git a/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1.noinstall b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1.noinstall new file mode 100644 index 0000000..c319c19 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{=example_recipe_name}}-0.1 diff --git a/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1/example.patch b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1/example.patch new file mode 100644 index 0000000..2000a34 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1/example.patch @@ -0,0 +1,12 @@ +# +# This is a non-functional placeholder file, here for example purposes +# only. +# +# If you had a patch for your recipe, you'd put it in this directory +# and reference it from your recipe's SRC_URI: +# +# SRC_URI += "file://example.patch" +# +# Note that you could also rename the directory containing this patch +# to remove the version number or simply rename it 'files'. Doing so +# allows you to use the same directory for multiple recipes. diff --git a/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1/helloworld.c b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1/helloworld.c new file mode 100644 index 0000000..71f2e46 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1/helloworld.c @@ -0,0 +1,8 @@ +#include <stdio.h> + +int main(int argc, char **argv) +{ + printf("Hello World!\n"); + + return 0; +} diff --git a/scripts/lib/bsp/substrate/target/arch/mips/.gitignore b/scripts/lib/bsp/substrate/target/arch/mips/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/.gitignore diff --git a/scripts/lib/bsp/substrate/target/arch/mips/conf/machine/machine.conf b/scripts/lib/bsp/substrate/target/arch/mips/conf/machine/machine.conf new file mode 100644 index 0000000..b319d62 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/conf/machine/machine.conf @@ -0,0 +1,39 @@ +# yocto-bsp-filename {{=machine}}.conf +#@TYPE: Machine +#@NAME: {{=machine}} + +#@DESCRIPTION: Machine configuration for {{=machine}} systems + +require conf/machine/include/tune-mips32.inc + +MACHINE_FEATURES = "screen keyboard pci usbhost ext2 ext3 serial" + +KERNEL_IMAGETYPE = "vmlinux" +KERNEL_ALT_IMAGETYPE = "vmlinux.bin" +KERNEL_IMAGE_STRIP_EXTRA_SECTIONS = ".comment" + +{{ if kernel_choice == "custom": preferred_kernel = "linux-yocto-custom" }} +{{ if kernel_choice == "linux-yocto-dev": preferred_kernel = "linux-yocto-dev" }} +{{ if kernel_choice == "custom" or kernel_choice == "linux-yocto-dev" : }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" + +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel = kernel_choice.split('_')[0] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel_version = kernel_choice.split('_')[1] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" +PREFERRED_VERSION_{{=preferred_kernel}} ?= "{{=preferred_kernel_version}}%" + +{{ input type:"boolean" name:"xserver" prio:"50" msg:"Do you need support for X? (y/n)" default:"y" }} +{{ if xserver == "y": }} +PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg" +XSERVER ?= "xserver-xorg \ + xf86-input-evdev \ + xf86-video-fbdev" + +SERIAL_CONSOLE = "115200 ttyS0" +USE_VT ?= "0" + +MACHINE_EXTRA_RRECOMMENDS = " kernel-modules" + +IMAGE_FSTYPES ?= "jffs2 tar.bz2" +JFFS2_ERASEBLOCK = "0x10000" diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files.noinstall b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files.noinstall new file mode 100644 index 0000000..1e0d92c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{ if kernel_choice != "custom": }} files diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-preempt-rt.scc b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-preempt-rt.scc new file mode 100644 index 0000000..a128255 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-preempt-rt.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-preempt-rt.scc +define KMACHINE {{=machine}} + +define KARCH mips + +include {{=map_preempt_rt_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-standard.scc b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-standard.scc new file mode 100644 index 0000000..7c9dc52 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-standard.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-standard.scc +define KMACHINE {{=machine}} + +define KARCH mips + +include {{=map_standard_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-tiny.scc b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-tiny.scc new file mode 100644 index 0000000..64f395b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-tiny.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-tiny.scc +define KMACHINE {{=machine}} + +define KARCH mips + +include {{=map_tiny_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-config.cfg b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-config.cfg new file mode 100644 index 0000000..47489e4 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-config.cfg @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-config.cfg diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-features.scc b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-features.scc new file mode 100644 index 0000000..85be26d --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-features.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-features.scc
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-patches.scc b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-patches.scc new file mode 100644 index 0000000..97f747f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-patches.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine.cfg b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine.cfg new file mode 100644 index 0000000..2fe4766 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine.cfg @@ -0,0 +1,2 @@ +# yocto-bsp-filename {{=machine}}.cfg +CONFIG_MIPS=y diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine.scc b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine.scc new file mode 100644 index 0000000..f39dc3e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine.scc @@ -0,0 +1,8 @@ +# yocto-bsp-filename {{=machine}}.scc +kconf hardware {{=machine}}.cfg + +include cfg/usb-mass-storage.scc +include cfg/fs/vfat.scc + +kconf hardware {{=machine}}-user-config.cfg +include {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/kernel-list.noinstall b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/kernel-list.noinstall new file mode 100644 index 0000000..00cf360 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/kernel-list.noinstall @@ -0,0 +1,5 @@ +{{ if kernel_choice != "custom": }} +{{ input type:"boolean" name:"use_default_kernel" prio:"10" msg:"Would you like to use the default (4.4) kernel? (y/n)" default:"y"}} + +{{ if kernel_choice != "custom" and use_default_kernel == "n": }} +{{ input type:"choicelist" name:"kernel_choice" gen:"bsp.kernel.kernels" prio:"10" msg:"Please choose the kernel to use in this BSP:" default:"linux-yocto_4.4"}} diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-dev.bbappend b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-dev.bbappend new file mode 100644 index 0000000..c336007 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-dev.bbappend @@ -0,0 +1,25 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-dev": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Would you like SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend new file mode 100644 index 0000000..0a47a4e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend new file mode 100644 index 0000000..815c77b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto_4.1.bbappend new file mode 100644 index 0000000..1e99a04 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto_4.1.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto_4.4.bbappend new file mode 100644 index 0000000..b88a06c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto_4.4.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/.gitignore b/scripts/lib/bsp/substrate/target/arch/mips64/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/.gitignore diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/conf/machine/machine.conf b/scripts/lib/bsp/substrate/target/arch/mips64/conf/machine/machine.conf new file mode 100644 index 0000000..3afc5e0 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/conf/machine/machine.conf @@ -0,0 +1,39 @@ +# yocto-bsp-filename {{=machine}}.conf +#@TYPE: Machine +#@NAME: {{=machine}} + +#@DESCRIPTION: Machine configuration for {{=machine}} systems + +require conf/machine/include/tune-mips64.inc + +MACHINE_FEATURES = "pci ext2 ext3 serial" + +KERNEL_IMAGETYPE = "vmlinux" +KERNEL_ALT_IMAGETYPE = "vmlinux.bin" +KERNEL_IMAGE_STRIP_EXTRA_SECTIONS = ".comment" + +{{ if kernel_choice == "custom": preferred_kernel = "linux-yocto-custom" }} +{{ if kernel_choice == "linux-yocto-dev": preferred_kernel = "linux-yocto-dev" }} +{{ if kernel_choice == "custom" or kernel_choice == "linux-yocto-dev" : }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" + +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel = kernel_choice.split('_')[0] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel_version = kernel_choice.split('_')[1] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" +PREFERRED_VERSION_{{=preferred_kernel}} ?= "{{=preferred_kernel_version}}%" + +{{ input type:"boolean" name:"xserver" prio:"50" msg:"Do you need support for X? (y/n)" default:"y" }} +{{ if xserver == "y": }} +PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg" +XSERVER ?= "xserver-xorg \ + xf86-input-evdev \ + xf86-video-fbdev" + +SERIAL_CONSOLE = "115200 ttyS0" +USE_VT ?= "0" + +MACHINE_EXTRA_RRECOMMENDS = " kernel-modules" + +IMAGE_FSTYPES ?= "jffs2 tar.bz2" +JFFS2_ERASEBLOCK = "0x10000" diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files.noinstall b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files.noinstall new file mode 100644 index 0000000..1e0d92c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{ if kernel_choice != "custom": }} files diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-preempt-rt.scc b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-preempt-rt.scc new file mode 100644 index 0000000..a128255 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-preempt-rt.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-preempt-rt.scc +define KMACHINE {{=machine}} + +define KARCH mips + +include {{=map_preempt_rt_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-standard.scc b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-standard.scc new file mode 100644 index 0000000..7c9dc52 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-standard.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-standard.scc +define KMACHINE {{=machine}} + +define KARCH mips + +include {{=map_standard_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-tiny.scc b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-tiny.scc new file mode 100644 index 0000000..64f395b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-tiny.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-tiny.scc +define KMACHINE {{=machine}} + +define KARCH mips + +include {{=map_tiny_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-config.cfg b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-config.cfg new file mode 100644 index 0000000..69efdcc --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-config.cfg @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-config.cfg
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-features.scc b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-features.scc new file mode 100644 index 0000000..85be26d --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-features.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-features.scc
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-patches.scc b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-patches.scc new file mode 100644 index 0000000..4c59daa --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-patches.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-patches.scc
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine.cfg b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine.cfg new file mode 100644 index 0000000..0cc906b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine.cfg @@ -0,0 +1,66 @@ +# yocto-bsp-filename {{=machine}}.cfg +#SOC +CONFIG_CAVIUM_OCTEON_SOC=y +CONFIG_CAVIUM_CN63XXP1=y +CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE=2 + +#Kernel +CONFIG_SMP=y +CONFIG_NR_CPUS=32 +#Executable file formats +CONFIG_MIPS32_COMPAT=y +CONFIG_MIPS32_O32=y +CONFIG_MIPS32_N32=y + + +#PCI +CONFIG_PCI=y +CONFIG_PCI_MSI=y + +#I2C +CONFIG_I2C=y +CONFIG_I2C_OCTEON=y + +CONFIG_HW_RANDOM_OCTEON=y + +#SPI +CONFIG_SPI=y +CONFIG_SPI_OCTEON=y + +#Misc +CONFIG_EEPROM_AT24=y +CONFIG_EEPROM_AT25=y +CONFIG_OCTEON_WDT=y + +CONFIG_STAGING=y + +#Ethernet +CONFIG_OCTEON_ETHERNET=y +CONFIG_OCTEON_MGMT_ETHERNET=y +CONFIG_MDIO_OCTEON=y + +#PHY +CONFIG_MARVELL_PHY=y +CONFIG_BROADCOM_PHY=y +CONFIG_BCM87XX_PHY=y + + +#USB +CONFIG_USB=y +CONFIG_OCTEON_USB=y +CONFIG_USB_OCTEON_EHCI=y +CONFIG_USB_OCTEON_OHCI=y +CONFIG_USB_OCTEON2_COMMON=y + +CONFIG_MTD=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_CMDLINE_PARTS=y + +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_NR_UARTS=2 +CONFIG_SERIAL_8250_RUNTIME_UARTS=2 +CONFIG_SERIAL_8250_DW=y + diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine.scc b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine.scc new file mode 100644 index 0000000..f39dc3e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine.scc @@ -0,0 +1,8 @@ +# yocto-bsp-filename {{=machine}}.scc +kconf hardware {{=machine}}.cfg + +include cfg/usb-mass-storage.scc +include cfg/fs/vfat.scc + +kconf hardware {{=machine}}-user-config.cfg +include {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/kernel-list.noinstall b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/kernel-list.noinstall new file mode 100644 index 0000000..00cf360 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/kernel-list.noinstall @@ -0,0 +1,5 @@ +{{ if kernel_choice != "custom": }} +{{ input type:"boolean" name:"use_default_kernel" prio:"10" msg:"Would you like to use the default (4.4) kernel? (y/n)" default:"y"}} + +{{ if kernel_choice != "custom" and use_default_kernel == "n": }} +{{ input type:"choicelist" name:"kernel_choice" gen:"bsp.kernel.kernels" prio:"10" msg:"Please choose the kernel to use in this BSP:" default:"linux-yocto_4.4"}} diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-dev.bbappend b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-dev.bbappend new file mode 100644 index 0000000..c336007 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-dev.bbappend @@ -0,0 +1,25 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-dev": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Would you like SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend new file mode 100644 index 0000000..0a47a4e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend new file mode 100644 index 0000000..815c77b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto_4.1.bbappend new file mode 100644 index 0000000..01a046c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto_4.1.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/edgerouter" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/edgerouter" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto_4.4.bbappend new file mode 100644 index 0000000..57c90fa --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto_4.4.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/edgerouter" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/edgerouter" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/.gitignore b/scripts/lib/bsp/substrate/target/arch/powerpc/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/.gitignore diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/conf/machine/machine.conf b/scripts/lib/bsp/substrate/target/arch/powerpc/conf/machine/machine.conf new file mode 100644 index 0000000..583c5e4 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/conf/machine/machine.conf @@ -0,0 +1,87 @@ +# yocto-bsp-filename {{=machine}}.conf +#@TYPE: Machine +#@NAME: {{=machine}} + +#@DESCRIPTION: Machine configuration for {{=machine}} systems + +TARGET_FPU = "" + +{{ input type:"choicelist" name:"tunefile" prio:"40" msg:"Which machine tuning would you like to use?" default:"tune_ppce300c3" }} +{{ input type:"choice" val:"tune_ppc476" msg:"ppc476 tuning optimizations" }} +{{ input type:"choice" val:"tune_ppc603e" msg:"ppc603e tuning optimizations" }} +{{ input type:"choice" val:"tune_ppc7400" msg:"ppc7400 tuning optimizations" }} +{{ input type:"choice" val:"tune_ppce300c2" msg:"ppce300c2 tuning optimizations" }} +{{ input type:"choice" val:"tune_ppce300c3" msg:"ppce300c3 tuning optimizations" }} +{{ input type:"choice" val:"tune_ppce500" msg:"ppce500 tuning optimizations" }} +{{ input type:"choice" val:"tune_ppce500mc" msg:"ppce500mc tuning optimizations" }} +{{ input type:"choice" val:"tune_ppce500v2" msg:"ppce500v2 tuning optimizations" }} +{{ input type:"choice" val:"tune_ppce5500" msg:"ppce5500 tuning optimizations" }} +{{ input type:"choice" val:"tune_ppce6500" msg:"ppce6500 tuning optimizations" }} +{{ input type:"choice" val:"tune_power5" msg:"power5 tuning optimizations" }} +{{ input type:"choice" val:"tune_power6" msg:"power6 tuning optimizations" }} +{{ input type:"choice" val:"tune_power7" msg:"power7 tuning optimizations" }} +{{ if tunefile == "tune_ppc476": }} +include conf/machine/include/tune-ppc476.inc +{{ if tunefile == "tune_ppc603e": }} +include conf/machine/include/tune-ppc603e.inc +{{ if tunefile == "tune_ppc7400": }} +include conf/machine/include/tune-ppc7400.inc +{{ if tunefile == "tune_ppce300c2": }} +include conf/machine/include/tune-ppce300c2.inc +{{ if tunefile == "tune_ppce300c3": }} +include conf/machine/include/tune-ppce300c3.inc +{{ if tunefile == "tune_ppce500": }} +include conf/machine/include/tune-ppce500.inc +{{ if tunefile == "tune_ppce500mc": }} +include conf/machine/include/tune-ppce500mc.inc +{{ if tunefile == "tune_ppce500v2": }} +include conf/machine/include/tune-ppce500v2.inc +{{ if tunefile == "tune_ppce5500": }} +include conf/machine/include/tune-ppce5500.inc +{{ if tunefile == "tune_ppce6500": }} +include conf/machine/include/tune-ppce6500.inc +{{ if tunefile == "tune_power5": }} +include conf/machine/include/tune-power5.inc +{{ if tunefile == "tune_power6": }} +include conf/machine/include/tune-power6.inc +{{ if tunefile == "tune_power7": }} +include conf/machine/include/tune-power7.inc + +KERNEL_IMAGETYPE = "uImage" + +EXTRA_IMAGEDEPENDS += "u-boot" +UBOOT_MACHINE_{{=machine}} = "MPC8315ERDB_config" + +SERIAL_CONSOLE = "115200 ttyS0" + +MACHINE_FEATURES = "keyboard pci ext2 ext3 serial" + +{{ if kernel_choice == "custom": preferred_kernel = "linux-yocto-custom" }} +{{ if kernel_choice == "linux-yocto-dev": preferred_kernel = "linux-yocto-dev" }} +{{ if kernel_choice == "custom" or kernel_choice == "linux-yocto-dev" : }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" + +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel = kernel_choice.split('_')[0] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel_version = kernel_choice.split('_')[1] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" +PREFERRED_VERSION_{{=preferred_kernel}} ?= "{{=preferred_kernel_version}}%" + +{{ input type:"boolean" name:"xserver" prio:"50" msg:"Do you need support for X? (y/n)" default:"y" }} +{{ if xserver == "y": }} +PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg" +XSERVER ?= "xserver-xorg \ + xf86-input-evdev \ + xf86-video-fbdev" + +PREFERRED_VERSION_u-boot ?= "v2016.01%" +{{ input type:"edit" name:"uboot_entrypoint" prio:"40" msg:"Please specify a value for UBOOT_ENTRYPOINT:" default:"0x00000000" }} +UBOOT_ENTRYPOINT = "{{=uboot_entrypoint}}" + +{{ input type:"edit" name:"kernel_devicetree" prio:"40" msg:"Please specify a [arch/powerpc/boot/dts/xxx] value for KERNEL_DEVICETREE:" default:"mpc8315erdb.dts" }} +KERNEL_DEVICETREE = "${S}/arch/powerpc/boot/dts/{{=kernel_devicetree}}" + +MACHINE_EXTRA_RRECOMMENDS = " kernel-modules" + +IMAGE_FSTYPES ?= "jffs2 tar.bz2" +JFFS2_ERASEBLOCK = "0x4000" diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files.noinstall b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files.noinstall new file mode 100644 index 0000000..1e0d92c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{ if kernel_choice != "custom": }} files diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-preempt-rt.scc b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-preempt-rt.scc new file mode 100644 index 0000000..91ccfb8 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-preempt-rt.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-preempt-rt.scc +define KMACHINE {{=machine}} + +define KARCH powerpc + +include {{=map_preempt_rt_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-standard.scc b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-standard.scc new file mode 100644 index 0000000..89b344f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-standard.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-standard.scc +define KMACHINE {{=machine}} + +define KARCH powerpc + +include {{=map_standard_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-tiny.scc b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-tiny.scc new file mode 100644 index 0000000..2701fd8 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-tiny.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-tiny.scc +define KMACHINE {{=machine}} + +define KARCH powerpc + +include {{=map_tiny_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-config.cfg b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-config.cfg new file mode 100644 index 0000000..47489e4 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-config.cfg @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-config.cfg diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-features.scc b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-features.scc new file mode 100644 index 0000000..582759e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-features.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-features.scc diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-patches.scc b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-patches.scc new file mode 100644 index 0000000..97f747f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-patches.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine.cfg b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine.cfg new file mode 100644 index 0000000..5bfe1fe --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine.cfg @@ -0,0 +1,164 @@ +# yocto-bsp-filename {{=machine}}.cfg +.......................................................................... +. WARNING +. +. This file is a kernel configuration fragment, and not a full kernel +. configuration file. The final kernel configuration is made up of +. an assembly of processed fragments, each of which is designed to +. capture a specific part of the final configuration (e.g. platform +. configuration, feature configuration, and board specific hardware +. configuration). For more information on kernel configuration, please +. consult the product documentation. +. +.......................................................................... +CONFIG_PPC32=y +CONFIG_PPC_OF=y +CONFIG_PPC_UDBG_16550=y + +# +# Processor support +# +CONFIG_PPC_83xx=y + +# +# Platform support +# +CONFIG_MPC831x_RDB=y +# CONFIG_PPC_CHRP is not set +# CONFIG_PPC_PMAC is not set + +# +# Bus options +# +CONFIG_PCI=y + +# +# Memory Technology Devices (MTD) +# +CONFIG_MTD=y +CONFIG_MTD_PARTITIONS=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_OF_PARTS=y + +# +# User Modules And Translation Layers +# +CONFIG_MTD_CHAR=y +CONFIG_MTD_BLOCK=y + +# +# RAM/ROM/Flash chip drivers +# +CONFIG_MTD_CFI=y +CONFIG_MTD_CFI_AMDSTD=y + +# +# Mapping drivers for chip access +# +CONFIG_MTD_PHYSMAP_OF=y + +# +# NAND Flash Device Drivers +# +CONFIG_MTD_NAND=y + +# +# Ethernet (1000 Mbit) +# +CONFIG_GIANFAR=y + +# +# Serial drivers +# +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_NR_UARTS=2 + +# +# Watchdog Device Drivers +# +CONFIG_8xxx_WDT=y + +# +# I2C support +# +CONFIG_I2C=y +CONFIG_I2C_CHARDEV=y + +# +# I2C Hardware Bus support +# +CONFIG_I2C_MPC=y + +CONFIG_SENSORS_LM75=y + +CONFIG_MISC_DEVICES=y + +# +# Miscellaneous I2C Chip support +# +CONFIG_EEPROM_AT24=y + +# +# SPI support +# +CONFIG_SPI=y +# CONFIG_SPI_DEBUG is not set +CONFIG_SPI_MASTER=y + +# +# SPI Master Controller Drivers +# +CONFIG_SPI_MPC8xxx=y + +# +# SPI Protocol Masters +# +CONFIG_HWMON=y + +# +# SCSI device support +# +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_SG=y +CONFIG_SCSI_LOGGING=y + +CONFIG_ATA=y +CONFIG_ATA_VERBOSE_ERROR=y +CONFIG_SATA_FSL=y +CONFIG_ATA_SFF=y + +# +# USB support +# +CONFIG_USB=m +CONFIG_USB_DEVICEFS=y + +# +# USB Host Controller Drivers +# +CONFIG_USB_EHCI_HCD=m +CONFIG_USB_EHCI_FSL=y +CONFIG_USB_STORAGE=m + +# +# Real Time Clock +# +CONFIG_RTC_CLASS=y + +# +# I2C RTC drivers +# +CONFIG_RTC_DRV_DS1307=y + +CONFIG_KGDB_8250=m + +CONFIG_CRYPTO_DEV_TALITOS=m + +CONFIG_FSL_DMA=y + +CONFIG_MMC=y +CONFIG_MMC_SPI=m + +CONFIG_USB_FSL_MPH_DR_OF=y diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine.scc b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine.scc new file mode 100644 index 0000000..7aac8b0 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine.scc @@ -0,0 +1,10 @@ +# yocto-bsp-filename {{=machine}}.scc +kconf hardware {{=machine}}.cfg + +include cfg/usb-mass-storage.scc +include cfg/fs/vfat.scc + +include cfg/dmaengine.scc + +kconf hardware {{=machine}}-user-config.cfg +include {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/kernel-list.noinstall b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/kernel-list.noinstall new file mode 100644 index 0000000..00cf360 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/kernel-list.noinstall @@ -0,0 +1,5 @@ +{{ if kernel_choice != "custom": }} +{{ input type:"boolean" name:"use_default_kernel" prio:"10" msg:"Would you like to use the default (4.4) kernel? (y/n)" default:"y"}} + +{{ if kernel_choice != "custom" and use_default_kernel == "n": }} +{{ input type:"choicelist" name:"kernel_choice" gen:"bsp.kernel.kernels" prio:"10" msg:"Please choose the kernel to use in this BSP:" default:"linux-yocto_4.4"}} diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-dev.bbappend b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-dev.bbappend new file mode 100644 index 0000000..c336007 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-dev.bbappend @@ -0,0 +1,25 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-dev": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Would you like SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend new file mode 100644 index 0000000..0a47a4e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend new file mode 100644 index 0000000..815c77b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto_4.1.bbappend new file mode 100644 index 0000000..1e99a04 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto_4.1.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto_4.4.bbappend new file mode 100644 index 0000000..b88a06c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto_4.4.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/conf/machine/machine.conf b/scripts/lib/bsp/substrate/target/arch/qemu/conf/machine/machine.conf new file mode 100644 index 0000000..67e1cbd --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/conf/machine/machine.conf @@ -0,0 +1,74 @@ +# yocto-bsp-filename {{=machine}}.conf +#@TYPE: Machine +#@NAME: {{=machine}} + +#@DESCRIPTION: Machine configuration for {{=machine}} systems + +{{ if kernel_choice == "custom": preferred_kernel = "linux-yocto-custom" }} +{{ if kernel_choice == "linux-yocto-dev": preferred_kernel = "linux-yocto-dev" }} +{{ if kernel_choice == "custom" or kernel_choice == "linux-yocto-dev" : }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" + +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel = kernel_choice.split('_')[0] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel_version = kernel_choice.split('_')[1] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" +PREFERRED_VERSION_{{=preferred_kernel}} ?= "{{=preferred_kernel_version}}%" + +{{ if qemuarch == "i386" or qemuarch == "x86_64": }} +PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg" +PREFERRED_PROVIDER_virtual/libgl ?= "mesa" +PREFERRED_PROVIDER_virtual/libgles1 ?= "mesa" +PREFERRED_PROVIDER_virtual/libgles2 ?= "mesa" + +{{ input type:"choicelist" name:"qemuarch" prio:"5" msg:"Which qemu architecture would you like to use?" default:"i386" }} +{{ input type:"choice" val:"i386" msg:"i386 (32-bit)" }} +{{ input type:"choice" val:"x86_64" msg:"x86_64 (64-bit)" }} +{{ input type:"choice" val:"arm" msg:"ARM (32-bit)" }} +{{ input type:"choice" val:"powerpc" msg:"PowerPC (32-bit)" }} +{{ input type:"choice" val:"mips" msg:"MIPS (32-bit)" }} +{{ input type:"choice" val:"mips64" msg:"MIPS64 (64-bit)" }} +{{ if qemuarch == "i386": }} +require conf/machine/include/qemu.inc +require conf/machine/include/tune-i586.inc +{{ if qemuarch == "x86_64": }} +require conf/machine/include/qemu.inc +DEFAULTTUNE ?= "core2-64" +require conf/machine/include/tune-core2.inc +{{ if qemuarch == "arm": }} +require conf/machine/include/qemu.inc +require conf/machine/include/tune-arm926ejs.inc +{{ if qemuarch == "powerpc": }} +require conf/machine/include/qemu.inc +require conf/machine/include/tune-ppc7400.inc +{{ if qemuarch == "mips": }} +require conf/machine/include/qemu.inc +require conf/machine/include/tune-mips32.inc +{{ if qemuarch == "mips64": }} +require conf/machine/include/qemu.inc +require conf/machine/include/tune-mips64.inc + +{{ if qemuarch == "i386" or qemuarch == "x86_64": }} +MACHINE_FEATURES += "x86" +KERNEL_IMAGETYPE = "bzImage" +SERIAL_CONSOLE = "115200 ttyS0" +XSERVER = "xserver-xorg \ + ${@bb.utils.contains('DISTRO_FEATURES', 'opengl', 'mesa-driver-swrast', '', d)} \ + xf86-input-vmmouse \ + xf86-input-keyboard \ + xf86-input-evdev \ + xf86-video-vmware" + +{{ if qemuarch == "arm": }} +KERNEL_IMAGETYPE = "zImage" +SERIAL_CONSOLE = "115200 ttyAMA0" + +{{ if qemuarch == "powerpc": }} +KERNEL_IMAGETYPE = "vmlinux" +SERIAL_CONSOLE = "115200 ttyS0" + +{{ if qemuarch == "mips" or qemuarch == "mips64": }} +KERNEL_IMAGETYPE = "vmlinux" +KERNEL_ALT_IMAGETYPE = "vmlinux.bin" +SERIAL_CONSOLE = "115200 ttyS0" +MACHINE_EXTRA_RRECOMMENDS = " kernel-modules" diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown/machine.noinstall b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown/machine.noinstall new file mode 100644 index 0000000..b442d02 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown/machine.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{=machine}} diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown/machine/interfaces b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown/machine/interfaces new file mode 100644 index 0000000..1696776 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown/machine/interfaces @@ -0,0 +1,5 @@ +# /etc/network/interfaces -- configuration file for ifup(8), ifdown(8) + +# The loopback interface +auto lo +iface lo inet loopback diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown_1.0.bbappend b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown_1.0.bbappend new file mode 100644 index 0000000..72d991c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown_1.0.bbappend @@ -0,0 +1 @@ +FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:" diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall new file mode 100644 index 0000000..b442d02 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{=machine}} diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf new file mode 100644 index 0000000..3bdde79 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf @@ -0,0 +1,77 @@ + +Section "Files" +EndSection + +Section "InputDevice" + Identifier "Generic Keyboard" + Driver "evdev" + Option "CoreKeyboard" + Option "Device" "/dev/input/by-path/platform-i8042-serio-0-event-kbd" + Option "XkbRules" "xorg" + Option "XkbModel" "evdev" + Option "XkbLayout" "us" +EndSection + +Section "InputDevice" + Identifier "Configured Mouse" +{{ if qemuarch == "arm" or qemuarch == "powerpc" or qemuarch == "mips" or qemuarch == "mips64": }} + Driver "mouse" +{{ if qemuarch == "i386" or qemuarch == "x86_64": }} + Driver "vmmouse" + + Option "CorePointer" + Option "Device" "/dev/input/mice" + Option "Protocol" "ImPS/2" + Option "ZAxisMapping" "4 5" + Option "Emulate3Buttons" "true" +EndSection + +Section "InputDevice" + Identifier "Qemu Tablet" + Driver "evdev" + Option "CorePointer" + Option "Device" "/dev/input/touchscreen0" + Option "USB" "on" +EndSection + +Section "Device" + Identifier "Graphics Controller" +{{ if qemuarch == "arm" or qemuarch == "powerpc" or qemuarch == "mips" or qemuarch == "mips64": }} + Driver "fbdev" +{{ if qemuarch == "i386" or qemuarch == "x86_64": }} + Driver "vmware" + +EndSection + +Section "Monitor" + Identifier "Generic Monitor" + Option "DPMS" + # 1024x600 59.85 Hz (CVT) hsync: 37.35 kHz; pclk: 49.00 MHz + Modeline "1024x600_60.00" 49.00 1024 1072 1168 1312 600 603 613 624 -hsync +vsync + # 640x480 @ 60Hz (Industry standard) hsync: 31.5kHz + ModeLine "640x480" 25.2 640 656 752 800 480 490 492 525 -hsync -vsync + # 640x480 @ 72Hz (VESA) hsync: 37.9kHz + ModeLine "640x480" 31.5 640 664 704 832 480 489 491 520 -hsync -vsync + # 640x480 @ 75Hz (VESA) hsync: 37.5kHz + ModeLine "640x480" 31.5 640 656 720 840 480 481 484 500 -hsync -vsync + # 640x480 @ 85Hz (VESA) hsync: 43.3kHz + ModeLine "640x480" 36.0 640 696 752 832 480 481 484 509 -hsync -vsync +EndSection + +Section "Screen" + Identifier "Default Screen" + Device "Graphics Controller" + Monitor "Generic Monitor" + SubSection "Display" + Modes "640x480" + EndSubSection +EndSection + +Section "ServerLayout" + Identifier "Default Layout" + Screen "Default Screen" + InputDevice "Generic Keyboard" + # InputDevice "Configured Mouse" + InputDevice "QEMU Tablet" + Option "AllowEmptyInput" "no" +EndSection diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend new file mode 100644 index 0000000..72d991c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend @@ -0,0 +1 @@ +FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:" diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files.noinstall b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files.noinstall new file mode 100644 index 0000000..0fb5283 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{ if kernel_choice != "custom": }} files
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-preempt-rt.scc b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-preempt-rt.scc new file mode 100644 index 0000000..a81b858 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-preempt-rt.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-preempt-rt.scc +define KMACHINE {{=machine}} + +define KARCH {{=qemuarch}} + +include {{=map_preempt_rt_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-standard.scc b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-standard.scc new file mode 100644 index 0000000..14554da --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-standard.scc @@ -0,0 +1,20 @@ +# yocto-bsp-filename {{=machine}}-standard.scc +define KMACHINE {{=machine}} + +define KARCH {{=qemuarch}} + +{{ if qemuarch == "i386" or qemuarch == "x86_64": }} +include {{=map_standard_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if qemuarch == "arm": }} +include bsp/arm-versatile-926ejs/arm-versatile-926ejs-standard +{{ if qemuarch == "powerpc": }} +include bsp/qemu-ppc32/qemu-ppc32-standard +{{ if qemuarch == "mips": }} +include bsp/mti-malta32/mti-malta32-be-standard +{{ if qemuarch == "mips64": }} +include bsp/mti-malta64/mti-malta64-be-standard +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-tiny.scc b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-tiny.scc new file mode 100644 index 0000000..41d4c6f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-tiny.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-tiny.scc +define KMACHINE {{=machine}} + +define KARCH {{=qemuarch}} + +include {{=map_tiny_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-config.cfg b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-config.cfg new file mode 100644 index 0000000..69efdcc --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-config.cfg @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-config.cfg
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-features.scc b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-features.scc new file mode 100644 index 0000000..582759e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-features.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-features.scc diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-patches.scc b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-patches.scc new file mode 100644 index 0000000..4c59daa --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-patches.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-patches.scc
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine.cfg b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine.cfg new file mode 100644 index 0000000..d560784 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine.cfg @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}.cfg
\ No newline at end of file diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine.scc b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine.scc new file mode 100644 index 0000000..8301e05 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine.scc @@ -0,0 +1,5 @@ +# yocto-bsp-filename {{=machine}}.scc +kconf hardware {{=machine}}.cfg + +kconf hardware {{=machine}}-user-config.cfg +include {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/kernel-list.noinstall b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/kernel-list.noinstall new file mode 100644 index 0000000..00cf360 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/kernel-list.noinstall @@ -0,0 +1,5 @@ +{{ if kernel_choice != "custom": }} +{{ input type:"boolean" name:"use_default_kernel" prio:"10" msg:"Would you like to use the default (4.4) kernel? (y/n)" default:"y"}} + +{{ if kernel_choice != "custom" and use_default_kernel == "n": }} +{{ input type:"choicelist" name:"kernel_choice" gen:"bsp.kernel.kernels" prio:"10" msg:"Please choose the kernel to use in this BSP:" default:"linux-yocto_4.4"}} diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-dev.bbappend b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-dev.bbappend new file mode 100644 index 0000000..7e3ce5b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-dev.bbappend @@ -0,0 +1,55 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-dev": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y" and qemuarch == "arm": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base your new BSP branch on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "arm": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose an existing machine branch to use for this BSP:" default:"standard/arm-versatile-926ejs" }} + +{{ if need_new_kbranch == "y" and qemuarch == "powerpc": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"powerpc" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "powerpc": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"powerpc" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/qemuppc" }} + +{{ if need_new_kbranch == "y" and qemuarch == "i386": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "i386": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "x86_64": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"x86_64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "x86_64": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"x86_64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "mips": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"mips" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "mips": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"mips" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/mti-malta32" }} + +{{ if need_new_kbranch == "y" and qemuarch == "mips64": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"mips64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "mips64": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"mips64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/mti-malta64" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Would you like SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend new file mode 100644 index 0000000..14ee16f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend @@ -0,0 +1,62 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y" and qemuarch == "arm": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"arm" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "arm": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"arm" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "powerpc": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"powerpc" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "powerpc": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"powerpc" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "i386": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "i386": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/common-pc" }} + +{{ if need_new_kbranch == "y" and qemuarch == "x86_64": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"x86_64" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "x86_64": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"x86_64" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "mips": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"mips" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "mips": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"mips" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "mips64": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"mips64" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "mips64": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"mips64" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend new file mode 100644 index 0000000..e256e08 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend @@ -0,0 +1,62 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y" and qemuarch == "arm": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"arm" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "arm": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"arm" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "powerpc": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"powerpc" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "powerpc": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"powerpc" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "i386": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "i386": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/common-pc" }} + +{{ if need_new_kbranch == "y" and qemuarch == "x86_64": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"x86_64" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "x86_64": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"x86_64" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "mips": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"mips" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "mips": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"mips" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "mips64": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"mips64" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "mips64": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"mips64" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto_4.1.bbappend new file mode 100644 index 0000000..fce67b4 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto_4.1.bbappend @@ -0,0 +1,61 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y" and qemuarch == "arm": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base your new BSP branch on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "arm": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose an existing machine branch to use for this BSP:" default:"standard/arm-versatile-926ejs" }} + +{{ if need_new_kbranch == "y" and qemuarch == "powerpc": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"powerpc" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "powerpc": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"powerpc" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/qemuppc" }} + +{{ if need_new_kbranch == "y" and qemuarch == "i386": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "i386": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "x86_64": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"x86_64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "x86_64": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"x86_64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "mips": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"mips" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/mti-malta32" }} + +{{ if need_new_kbranch == "n" and qemuarch == "mips64": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"mips64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/mti-malta64" }} + +{{ if need_new_kbranch == "y" and qemuarch == "mips": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"mips" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "mips64": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"mips64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Would you like SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto_4.4.bbappend new file mode 100644 index 0000000..4097932 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto_4.4.bbappend @@ -0,0 +1,61 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y" and qemuarch == "arm": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base your new BSP branch on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "arm": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose an existing machine branch to use for this BSP:" default:"standard/arm-versatile-926ejs" }} + +{{ if need_new_kbranch == "y" and qemuarch == "powerpc": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"powerpc" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "powerpc": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"powerpc" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/qemuppc" }} + +{{ if need_new_kbranch == "y" and qemuarch == "i386": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "i386": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "x86_64": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"x86_64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "x86_64": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"x86_64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n" and qemuarch == "mips": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"mips" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/mti-malta32" }} + +{{ if need_new_kbranch == "n" and qemuarch == "mips64": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"mips64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/mti-malta64" }} + +{{ if need_new_kbranch == "y" and qemuarch == "mips": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"mips" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "y" and qemuarch == "mips64": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"mips64" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Would you like SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/.gitignore b/scripts/lib/bsp/substrate/target/arch/x86_64/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/.gitignore diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/conf/machine/machine.conf b/scripts/lib/bsp/substrate/target/arch/x86_64/conf/machine/machine.conf new file mode 100644 index 0000000..e4b8251 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/conf/machine/machine.conf @@ -0,0 +1,65 @@ +# yocto-bsp-filename {{=machine}}.conf +#@TYPE: Machine +#@NAME: {{=machine}} + +#@DESCRIPTION: Machine configuration for {{=machine}} systems + +{{ if kernel_choice == "custom": preferred_kernel = "linux-yocto-custom" }} +{{ if kernel_choice == "linux-yocto-dev": preferred_kernel = "linux-yocto-dev" }} +{{ if kernel_choice == "custom" or kernel_choice == "linux-yocto-dev" : }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" + +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel = kernel_choice.split('_')[0] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": preferred_kernel_version = kernel_choice.split('_')[1] }} +{{ if kernel_choice != "custom" and kernel_choice != "linux-yocto-dev": }} +PREFERRED_PROVIDER_virtual/kernel ?= "{{=preferred_kernel}}" +PREFERRED_VERSION_{{=preferred_kernel}} ?= "{{=preferred_kernel_version}}%" + +{{ input type:"choicelist" name:"tunefile" prio:"40" msg:"Which machine tuning would you like to use?" default:"tune_core2" }} +{{ input type:"choice" val:"tune_core2" msg:"Core2 tuning optimizations" }} +{{ input type:"choice" val:"tune_corei7" msg:"Corei7 tuning optimizations" }} +{{ if tunefile == "tune_core2": }} +DEFAULTTUNE ?= "core2-64" +require conf/machine/include/tune-core2.inc +{{ if tunefile == "tune_corei7": }} +DEFAULTTUNE ?= "corei7-64" +require conf/machine/include/tune-corei7.inc + +require conf/machine/include/x86-base.inc + +MACHINE_FEATURES += "wifi efi pcbios" + +{{ input type:"boolean" name:"xserver" prio:"50" msg:"Do you need support for X? (y/n)" default:"y" }} + +{{ if xserver == "y": }} +{{ input type:"choicelist" name:"xserver_choice" prio:"50" msg:"Please select an xserver for this machine:" default:"xserver_i915" }} + +{{ input type:"choice" val:"xserver_vesa" msg:"VESA xserver support" }} +{{ input type:"choice" val:"xserver_i915" msg:"i915 xserver support" }} +{{ input type:"choice" val:"xserver_i965" msg:"i965 xserver support" }} +{{ input type:"choice" val:"xserver_fbdev" msg:"fbdev xserver support" }} +{{ input type:"choice" val:"xserver_modesetting" msg:"modesetting xserver support" }} +{{ if xserver == "y": }} +XSERVER ?= "${XSERVER_X86_BASE} \ + ${XSERVER_X86_EXT} \ +{{ if xserver == "y" and xserver_choice == "xserver_vesa": }} + ${XSERVER_X86_VESA} \ +{{ if xserver == "y" and xserver_choice == "xserver_i915": }} + ${XSERVER_X86_I915} \ +{{ if xserver == "y" and xserver_choice == "xserver_i965": }} + ${XSERVER_X86_I965} \ +{{ if xserver == "y" and xserver_choice == "xserver_fbdev": }} + ${XSERVER_X86_FBDEV} \ +{{ if xserver == "y" and xserver_choice == "xserver_modesetting": }} + ${XSERVER_X86_MODESETTING} \ +{{ if xserver == "y": }} + " + +MACHINE_EXTRA_RRECOMMENDS += "linux-firmware v86d eee-acpi-scripts" + +EXTRA_OECONF_append_pn-matchbox-panel-2 = " --with-battery=acpi" + +GLIBC_ADDONS = "nptl" + +{{ if xserver == "y" and xserver_choice == "xserver_vesa": }} +APPEND += "video=vesafb vga=0x318" diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall new file mode 100644 index 0000000..b442d02 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{=machine}} diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf new file mode 100644 index 0000000..ac9a0f1 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf @@ -0,0 +1 @@ +# yocto-bsp-filename {{ if xserver == "y": }} this diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend new file mode 100644 index 0000000..3083003 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend @@ -0,0 +1,2 @@ +# yocto-bsp-filename {{ if xserver == "y": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:" diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files.noinstall b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files.noinstall new file mode 100644 index 0000000..1e0d92c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files.noinstall @@ -0,0 +1 @@ +# yocto-bsp-dirname {{ if kernel_choice != "custom": }} files diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-preempt-rt.scc b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-preempt-rt.scc new file mode 100644 index 0000000..bbeeecd --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-preempt-rt.scc @@ -0,0 +1,17 @@ +# yocto-bsp-filename {{=machine}}-preempt-rt.scc +define KMACHINE {{=machine}} + +define KARCH x86_64 + +include {{=map_preempt_rt_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc + +# default policy for preempt-rt kernels +include cfg/usb-mass-storage.scc +include cfg/boot-live.scc +include features/latencytop/latencytop.scc +include features/profiling/profiling.scc diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-standard.scc b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-standard.scc new file mode 100644 index 0000000..9c9cc90 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-standard.scc @@ -0,0 +1,17 @@ +# yocto-bsp-filename {{=machine}}-standard.scc +define KMACHINE {{=machine}} + +define KARCH x86_64 + +include {{=map_standard_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc + +# default policy for standard kernels +include cfg/usb-mass-storage.scc +include cfg/boot-live.scc +include features/latencytop/latencytop.scc +include features/profiling/profiling.scc diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-tiny.scc b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-tiny.scc new file mode 100644 index 0000000..b53706f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-tiny.scc @@ -0,0 +1,11 @@ +# yocto-bsp-filename {{=machine}}-tiny.scc +define KMACHINE {{=machine}} + +define KARCH x86_64 + +include {{=map_tiny_kbranch(need_new_kbranch, new_kbranch, existing_kbranch)}} +{{ if need_new_kbranch == "y": }} +define KTYPE {{=new_kbranch}} +branch {{=machine}} + +include {{=machine}}.scc diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-config.cfg b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-config.cfg new file mode 100644 index 0000000..47489e4 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-config.cfg @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-config.cfg diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-features.scc b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-features.scc new file mode 100644 index 0000000..582759e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-features.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-features.scc diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-patches.scc b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-patches.scc new file mode 100644 index 0000000..97f747f --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-patches.scc @@ -0,0 +1 @@ +# yocto-bsp-filename {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine.cfg b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine.cfg new file mode 100644 index 0000000..3290dde --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine.cfg @@ -0,0 +1,48 @@ +# yocto-bsp-filename {{=machine}}.cfg +CONFIG_PRINTK=y + +# Basic hardware support for the box - network, USB, PCI, sound +CONFIG_NETDEVICES=y +CONFIG_ATA=y +CONFIG_ATA_GENERIC=y +CONFIG_ATA_SFF=y +CONFIG_PCI=y +CONFIG_MMC=y +CONFIG_MMC_SDHCI=y +CONFIG_USB_SUPPORT=y +CONFIG_USB=y +CONFIG_USB_ARCH_HAS_EHCI=y +CONFIG_R8169=y +CONFIG_PATA_SCH=y +CONFIG_MMC_SDHCI_PCI=y +CONFIG_USB_EHCI_HCD=y +CONFIG_PCIEPORTBUS=y +CONFIG_NET=y +CONFIG_USB_UHCI_HCD=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_SG=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_HDA_INTEL=y + +# Make sure these are on, otherwise the bootup won't be fun +CONFIG_EXT3_FS=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_MODULES=y +CONFIG_SHMEM=y +CONFIG_TMPFS=y +CONFIG_PACKET=y + +CONFIG_I2C=y +CONFIG_AGP=y +CONFIG_PM=y +CONFIG_ACPI=y +CONFIG_INPUT=y + +# Needed for booting (and using) USB memory sticks +CONFIG_BLK_DEV_LOOP=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y + +CONFIG_RD_GZIP=y diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine.scc b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine.scc new file mode 100644 index 0000000..9b7c291 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine.scc @@ -0,0 +1,14 @@ +# yocto-bsp-filename {{=machine}}.scc +kconf hardware {{=machine}}.cfg + +include features/serial/8250.scc +{{ if xserver == "y" and xserver_choice == "xserver_vesa": }} +include cfg/vesafb.scc +{{ if xserver == "y" and xserver_choice == "xserver_i915" or xserver_choice == "xserver_i965": }} +include features/i915/i915.scc + +include cfg/usb-mass-storage.scc +include features/power/intel.scc + +kconf hardware {{=machine}}-user-config.cfg +include {{=machine}}-user-patches.scc diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/kernel-list.noinstall b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/kernel-list.noinstall new file mode 100644 index 0000000..00cf360 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/kernel-list.noinstall @@ -0,0 +1,5 @@ +{{ if kernel_choice != "custom": }} +{{ input type:"boolean" name:"use_default_kernel" prio:"10" msg:"Would you like to use the default (4.4) kernel? (y/n)" default:"y"}} + +{{ if kernel_choice != "custom" and use_default_kernel == "n": }} +{{ input type:"choicelist" name:"kernel_choice" gen:"bsp.kernel.kernels" prio:"10" msg:"Please choose the kernel to use in this BSP:" default:"linux-yocto_4.4"}} diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-dev.bbappend b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-dev.bbappend new file mode 100644 index 0000000..c336007 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-dev.bbappend @@ -0,0 +1,25 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-dev": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" nameappend:"i386" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Would you like SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend new file mode 100644 index 0000000..0a47a4e --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend new file mode 100644 index 0000000..815c77b --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend @@ -0,0 +1,33 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto-tiny_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard/tiny" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/tiny/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-tiny.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-patches.scc \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto-tiny_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto_4.1.bbappend b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto_4.1.bbappend new file mode 100644 index 0000000..1e99a04 --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto_4.1.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.1": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.1" diff --git a/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto_4.4.bbappend b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto_4.4.bbappend new file mode 100644 index 0000000..b88a06c --- /dev/null +++ b/scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto_4.4.bbappend @@ -0,0 +1,32 @@ +# yocto-bsp-filename {{ if kernel_choice == "linux-yocto_4.4": }} this +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +PR := "${PR}.1" + +COMPATIBLE_MACHINE_{{=machine}} = "{{=machine}}" + +{{ input type:"boolean" name:"need_new_kbranch" prio:"20" msg:"Do you need a new machine branch for this BSP (the alternative is to re-use an existing branch)? [y/n]" default:"y" }} + +{{ if need_new_kbranch == "y": }} +{{ input type:"choicelist" name:"new_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +{{ input type:"choicelist" name:"existing_kbranch" gen:"bsp.kernel.all_branches" branches_base:"standard" prio:"20" msg:"Please choose a machine branch to base this BSP on:" default:"standard/base" }} + +{{ if need_new_kbranch == "n": }} +KBRANCH_{{=machine}} = "{{=existing_kbranch}}" + +{{ input type:"boolean" name:"smp" prio:"30" msg:"Do you need SMP support? (y/n)" default:"y"}} +{{ if smp == "y": }} +KERNEL_FEATURES_append_{{=machine}} += " cfg/smp.scc" + +SRC_URI += "file://{{=machine}}-standard.scc \ + file://{{=machine}}-user-config.cfg \ + file://{{=machine}}-user-features.scc \ + " + +# replace these SRCREVs with the real commit ids once you've had +# the appropriate changes committed to the upstream linux-yocto repo +SRCREV_machine_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +SRCREV_meta_pn-linux-yocto_{{=machine}} ?= "${AUTOREV}" +#LINUX_VERSION = "4.4" diff --git a/scripts/lib/bsp/tags.py b/scripts/lib/bsp/tags.py new file mode 100644 index 0000000..3719427 --- /dev/null +++ b/scripts/lib/bsp/tags.py @@ -0,0 +1,49 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This module provides a place to define common constants for the +# Yocto BSP Tools. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] intel.com> +# + +OPEN_TAG = "{{" +CLOSE_TAG = "}}" +ASSIGN_TAG = "{{=" +INPUT_TAG = "input" +IF_TAG = "if" +FILENAME_TAG = "yocto-bsp-filename" +DIRNAME_TAG = "yocto-bsp-dirname" + +INDENT_STR = " " + +BLANKLINE_STR = "of.write(\"\\n\")" +NORMAL_START = "of.write" +OPEN_START = "current_file =" + +INPUT_TYPE_PROPERTY = "type" + +SRC_URI_FILE = "file://" + +GIT_CHECK_URI = "git://git.yoctoproject.org/linux-yocto-dev.git" + + + diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py new file mode 100644 index 0000000..ff97dfc --- /dev/null +++ b/scripts/lib/devtool/__init__.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python + +# Development tool - utility functions for plugins +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool plugins module""" + +import os +import sys +import subprocess +import logging +import re + +logger = logging.getLogger('devtool') + + +class DevtoolError(Exception): + """Exception for handling devtool errors""" + pass + + +def exec_build_env_command(init_path, builddir, cmd, watch=False, **options): + """Run a program in bitbake build context""" + import bb + if not 'cwd' in options: + options["cwd"] = builddir + if init_path: + # As the OE init script makes use of BASH_SOURCE to determine OEROOT, + # and can't determine it when running under dash, we need to set + # the executable to bash to correctly set things up + if not 'executable' in options: + options['executable'] = 'bash' + logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path)) + init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir) + else: + logger.debug('Executing command "%s"' % cmd) + init_prefix = '' + if watch: + if sys.stdout.isatty(): + # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly) + cmd = 'script -e -q -c "%s" /dev/null' % cmd + return exec_watch('%s%s' % (init_prefix, cmd), **options) + else: + return bb.process.run('%s%s' % (init_prefix, cmd), **options) + +def exec_watch(cmd, **options): + """Run program with stdout shown on sys.stdout""" + import bb + if isinstance(cmd, basestring) and not "shell" in options: + options["shell"] = True + + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options + ) + + buf = '' + while True: + out = process.stdout.read(1) + if out: + sys.stdout.write(out) + sys.stdout.flush() + buf += out + elif out == '' and process.poll() != None: + break + + if process.returncode != 0: + raise bb.process.ExecutionError(cmd, process.returncode, buf, None) + + return buf, None + +def exec_fakeroot(d, cmd, **kwargs): + """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions""" + # Grab the command and check it actually exists + fakerootcmd = d.getVar('FAKEROOTCMD', True) + if not os.path.exists(fakerootcmd): + logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built') + return 2 + # Set up the appropriate environment + newenv = dict(os.environ) + fakerootenv = d.getVar('FAKEROOTENV', True) + for varvalue in fakerootenv.split(): + if '=' in varvalue: + splitval = varvalue.split('=', 1) + newenv[splitval[0]] = splitval[1] + return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs) + +def setup_tinfoil(config_only=False, basepath=None, tracking=False): + """Initialize tinfoil api from bitbake""" + import scriptpath + orig_cwd = os.path.abspath(os.curdir) + try: + if basepath: + os.chdir(basepath) + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + sys.exit(1) + + import bb.tinfoil + tinfoil = bb.tinfoil.Tinfoil(tracking=tracking) + tinfoil.prepare(config_only) + tinfoil.logger.setLevel(logger.getEffectiveLevel()) + finally: + os.chdir(orig_cwd) + return tinfoil + +def get_recipe_file(cooker, pn): + """Find recipe file corresponding a package name""" + import oe.recipeutils + recipefile = oe.recipeutils.pn_to_recipe(cooker, pn) + if not recipefile: + skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn) + if skipreasons: + logger.error('\n'.join(skipreasons)) + else: + logger.error("Unable to find any recipe file matching %s" % pn) + return recipefile + +def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True): + """Parse recipe of a package""" + import oe.recipeutils + recipefile = get_recipe_file(tinfoil.cooker, pn) + if not recipefile: + # Error already logged + return None + if appends: + append_files = tinfoil.cooker.collection.get_file_appends(recipefile) + if filter_workspace: + # Filter out appends from the workspace + append_files = [path for path in append_files if + not path.startswith(config.workspace_path)] + else: + append_files = None + return oe.recipeutils.parse_recipe(recipefile, append_files, + tinfoil.config_data) + +def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False): + """ + Check that a recipe is in the workspace and (optionally) that source + is present. + """ + + workspacepn = pn + + for recipe, value in workspace.iteritems(): + if recipe == pn: + break + if bbclassextend: + recipefile = value['recipefile'] + if recipefile: + targets = get_bbclassextend_targets(recipefile, recipe) + if pn in targets: + workspacepn = recipe + break + else: + raise DevtoolError("No recipe named '%s' in your workspace" % pn) + + if checksrc: + srctree = workspace[workspacepn]['srctree'] + if not os.path.exists(srctree): + raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn)) + if not os.listdir(srctree): + raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn)) + + return workspacepn + +def use_external_build(same_dir, no_same_dir, d): + """ + Determine if we should use B!=S (separate build and source directories) or not + """ + b_is_s = True + if no_same_dir: + logger.info('Using separate build directory since --no-same-dir specified') + b_is_s = False + elif same_dir: + logger.info('Using source tree as build directory since --same-dir specified') + elif bb.data.inherits_class('autotools-brokensep', d): + logger.info('Using source tree as build directory since recipe inherits autotools-brokensep') + elif d.getVar('B', True) == os.path.abspath(d.getVar('S', True)): + logger.info('Using source tree as build directory since that would be the default for this recipe') + else: + b_is_s = False + return b_is_s + +def setup_git_repo(repodir, version, devbranch, basetag='devtool-base'): + """ + Set up the git repository for the source tree + """ + import bb.process + if not os.path.exists(os.path.join(repodir, '.git')): + bb.process.run('git init', cwd=repodir) + bb.process.run('git add .', cwd=repodir) + commit_cmd = ['git', 'commit', '-q'] + stdout, _ = bb.process.run('git status --porcelain', cwd=repodir) + if not stdout: + commit_cmd.append('--allow-empty') + commitmsg = "Initial empty commit with no upstream sources" + elif version: + commitmsg = "Initial commit from upstream at version %s" % version + else: + commitmsg = "Initial commit from upstream" + commit_cmd += ['-m', commitmsg] + bb.process.run(commit_cmd, cwd=repodir) + + bb.process.run('git checkout -b %s' % devbranch, cwd=repodir) + bb.process.run('git tag -f %s' % basetag, cwd=repodir) + +def recipe_to_append(recipefile, config, wildcard=False): + """ + Convert a recipe file to a bbappend file path within the workspace. + NOTE: if the bbappend already exists, you should be using + workspace[args.recipename]['bbappend'] instead of calling this + function. + """ + appendname = os.path.splitext(os.path.basename(recipefile))[0] + if wildcard: + appendname = re.sub(r'_.*', '_%', appendname) + appendpath = os.path.join(config.workspace_path, 'appends') + appendfile = os.path.join(appendpath, appendname + '.bbappend') + return appendfile + +def get_bbclassextend_targets(recipefile, pn): + """ + Cheap function to get BBCLASSEXTEND and then convert that to the + list of targets that would result. + """ + import bb.utils + + values = {} + def get_bbclassextend_varfunc(varname, origvalue, op, newlines): + values[varname] = origvalue + return origvalue, None, 0, True + with open(recipefile, 'r') as f: + bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc) + + targets = [] + bbclassextend = values.get('BBCLASSEXTEND', '').split() + if bbclassextend: + for variant in bbclassextend: + if variant == 'nativesdk': + targets.append('%s-%s' % (variant, pn)) + elif variant in ['native', 'cross', 'crosssdk']: + targets.append('%s-%s' % (pn, variant)) + return targets diff --git a/scripts/lib/devtool/build.py b/scripts/lib/devtool/build.py new file mode 100644 index 0000000..48f6fe1 --- /dev/null +++ b/scripts/lib/devtool/build.py @@ -0,0 +1,86 @@ +# Development tool - build command plugin +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool build plugin""" + +import os +import bb +import logging +import argparse +import tempfile +from devtool import exec_build_env_command, check_workspace_recipe, DevtoolError + +logger = logging.getLogger('devtool') + + +def _set_file_values(fn, values): + remaining = values.keys() + + def varfunc(varname, origvalue, op, newlines): + newvalue = values.get(varname, origvalue) + remaining.remove(varname) + return (newvalue, '=', 0, True) + + with open(fn, 'r') as f: + (updated, newlines) = bb.utils.edit_metadata(f, values, varfunc) + + for item in remaining: + updated = True + newlines.append('%s = "%s"' % (item, values[item])) + + if updated: + with open(fn, 'w') as f: + f.writelines(newlines) + return updated + +def _get_build_tasks(config): + tasks = config.get('Build', 'build_task', 'populate_sysroot,packagedata').split(',') + return ['do_%s' % task.strip() for task in tasks] + +def build(args, config, basepath, workspace): + """Entry point for the devtool 'build' subcommand""" + workspacepn = check_workspace_recipe(workspace, args.recipename, bbclassextend=True) + + build_tasks = _get_build_tasks(config) + + bbappend = workspace[workspacepn]['bbappend'] + if args.disable_parallel_make: + logger.info("Disabling 'make' parallelism") + _set_file_values(bbappend, {'PARALLEL_MAKE': ''}) + try: + bbargs = [] + for task in build_tasks: + if args.recipename.endswith('-native') and 'package' in task: + continue + bbargs.append('%s:%s' % (args.recipename, task)) + exec_build_env_command(config.init_path, basepath, 'bitbake %s' % ' '.join(bbargs), watch=True) + except bb.process.ExecutionError as e: + # We've already seen the output since watch=True, so just ensure we return something to the user + return e.exitcode + finally: + if args.disable_parallel_make: + _set_file_values(bbappend, {'PARALLEL_MAKE': None}) + + return 0 + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + parser_build = subparsers.add_parser('build', help='Build a recipe', + description='Builds the specified recipe using bitbake (up to and including %s)' % ', '.join(_get_build_tasks(context.config)), + group='working') + parser_build.add_argument('recipename', help='Recipe to build') + parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism') + parser_build.set_defaults(func=build) diff --git a/scripts/lib/devtool/build_image.py b/scripts/lib/devtool/build_image.py new file mode 100644 index 0000000..14c646a --- /dev/null +++ b/scripts/lib/devtool/build_image.py @@ -0,0 +1,168 @@ +# Development tool - build-image plugin +# +# Copyright (C) 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +"""Devtool plugin containing the build-image subcommand.""" + +import os +import errno +import logging + +from bb.process import ExecutionError +from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError + +logger = logging.getLogger('devtool') + +class TargetNotImageError(Exception): + pass + +def _get_packages(tinfoil, workspace, config): + """Get list of packages from recipes in the workspace.""" + result = [] + for recipe in workspace: + data = parse_recipe(config, tinfoil, recipe, True) + if 'class-target' in data.getVar('OVERRIDES', True).split(':'): + if recipe in data.getVar('PACKAGES', True).split(): + result.append(recipe) + else: + logger.warning("Skipping recipe %s as it doesn't produce a " + "package with the same name", recipe) + return result + +def build_image(args, config, basepath, workspace): + """Entry point for the devtool 'build-image' subcommand.""" + + image = args.imagename + auto_image = False + if not image: + sdk_targets = config.get('SDK', 'sdk_targets', '').split() + if sdk_targets: + image = sdk_targets[0] + auto_image = True + if not image: + raise DevtoolError('Unable to determine image to build, please specify one') + + try: + if args.add_packages: + add_packages = args.add_packages.split(',') + else: + add_packages = None + result, outputdir = build_image_task(config, basepath, workspace, image, add_packages) + except TargetNotImageError: + if auto_image: + raise DevtoolError('Unable to determine image to build, please specify one') + else: + raise DevtoolError('Specified recipe %s is not an image recipe' % image) + + if result == 0: + logger.info('Successfully built %s. You can find output files in %s' + % (image, outputdir)) + return result + +def build_image_task(config, basepath, workspace, image, add_packages=None, task=None, extra_append=None): + # remove <image>.bbappend to make sure setup_tinfoil doesn't + # break because of it + target_basename = config.get('SDK', 'target_basename', '') + if target_basename: + appendfile = os.path.join(config.workspace_path, 'appends', + '%s.bbappend' % target_basename) + try: + os.unlink(appendfile) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + + tinfoil = setup_tinfoil(basepath=basepath) + rd = parse_recipe(config, tinfoil, image, True) + if not rd: + # Error already shown + return (1, None) + if not bb.data.inherits_class('image', rd): + raise TargetNotImageError() + + # Get the actual filename used and strip the .bb and full path + target_basename = rd.getVar('FILE', True) + target_basename = os.path.splitext(os.path.basename(target_basename))[0] + config.set('SDK', 'target_basename', target_basename) + config.write() + + appendfile = os.path.join(config.workspace_path, 'appends', + '%s.bbappend' % target_basename) + + outputdir = None + try: + if workspace or add_packages: + if add_packages: + packages = add_packages + else: + packages = _get_packages(tinfoil, workspace, config) + else: + packages = None + if not task: + if not packages and not add_packages and workspace: + logger.warning('No recipes in workspace, building image %s unmodified', image) + elif not packages: + logger.warning('No packages to add, building image %s unmodified', image) + + if packages or extra_append: + bb.utils.mkdirhier(os.path.dirname(appendfile)) + with open(appendfile, 'w') as afile: + if packages: + # include packages from workspace recipes into the image + afile.write('IMAGE_INSTALL_append = " %s"\n' % ' '.join(packages)) + if not task: + logger.info('Building image %s with the following ' + 'additional packages: %s', image, ' '.join(packages)) + if extra_append: + for line in extra_append: + afile.write('%s\n' % line) + + if task in ['populate_sdk', 'populate_sdk_ext']: + outputdir = rd.getVar('SDK_DEPLOY', True) + else: + outputdir = rd.getVar('DEPLOY_DIR_IMAGE', True) + + tinfoil.shutdown() + + options = '' + if task: + options += '-c %s' % task + + # run bitbake to build image (or specified task) + try: + exec_build_env_command(config.init_path, basepath, + 'bitbake %s %s' % (options, image), watch=True) + except ExecutionError as err: + return (err.exitcode, None) + finally: + if os.path.isfile(appendfile): + os.unlink(appendfile) + return (0, outputdir) + + +def register_commands(subparsers, context): + """Register devtool subcommands from the build-image plugin""" + parser = subparsers.add_parser('build-image', + help='Build image including workspace recipe packages', + description='Builds an image, extending it to include ' + 'packages from recipes in the workspace', + group='testbuild', order=-10) + parser.add_argument('imagename', help='Image recipe to build', nargs='?') + parser.add_argument('-p', '--add-packages', help='Instead of adding packages for the ' + 'entire workspace, specify packages to be added to the image ' + '(separate multiple packages by commas)', + metavar='PACKAGES') + parser.set_defaults(func=build_image) diff --git a/scripts/lib/devtool/build_sdk.py b/scripts/lib/devtool/build_sdk.py new file mode 100644 index 0000000..b89d65b --- /dev/null +++ b/scripts/lib/devtool/build_sdk.py @@ -0,0 +1,65 @@ +# Development tool - build-sdk command plugin +# +# Copyright (C) 2015-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import subprocess +import logging +import glob +import shutil +import errno +import sys +import tempfile +from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError +from devtool import build_image + +logger = logging.getLogger('devtool') + + +def build_sdk(args, config, basepath, workspace): + """Entry point for the devtool build-sdk command""" + + sdk_targets = config.get('SDK', 'sdk_targets', '').split() + if sdk_targets: + image = sdk_targets[0] + else: + raise DevtoolError('Unable to determine image to build SDK for') + + extra_append = ['SDK_DERIVATIVE = "1"'] + try: + result, outputdir = build_image.build_image_task(config, + basepath, + workspace, + image, + task='populate_sdk_ext', + extra_append=extra_append) + except build_image.TargetNotImageError: + raise DevtoolError('Unable to determine image to build SDK for') + + if result == 0: + logger.info('Successfully built SDK. You can find output files in %s' + % outputdir) + return result + + +def register_commands(subparsers, context): + """Register devtool subcommands""" + if context.fixed_setup: + parser_build_sdk = subparsers.add_parser('build-sdk', + help='Build a derivative SDK of this one', + description='Builds an extensible SDK based upon this one and the items in your workspace', + group='advanced') + parser_build_sdk.set_defaults(func=build_sdk) diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py new file mode 100644 index 0000000..66644cc --- /dev/null +++ b/scripts/lib/devtool/deploy.py @@ -0,0 +1,304 @@ +# Development tool - deploy/undeploy command plugin +# +# Copyright (C) 2014-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool plugin containing the deploy subcommands""" + +import os +import subprocess +import logging +import tempfile +import shutil +import argparse_oe +from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError + +logger = logging.getLogger('devtool') + +deploylist_path = '/.devtool' + +def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False): + """ + Prepare a shell script for running on the target to + deploy/undeploy files. We have to be careful what we put in this + script - only commands that are likely to be available on the + target are suitable (the target might be constrained, e.g. using + busybox rather than bash with coreutils). + """ + lines = [] + lines.append('#!/bin/sh') + lines.append('set -e') + if undeployall: + # Yes, I know this is crude - but it does work + lines.append('for entry in %s/*.list; do' % deploylist_path) + lines.append('[ ! -f $entry ] && exit') + lines.append('set `basename $entry | sed "s/.list//"`') + if dryrun: + if not deploy: + lines.append('echo "Previously deployed files for $1:"') + lines.append('manifest="%s/$1.list"' % deploylist_path) + lines.append('preservedir="%s/$1.preserve"' % deploylist_path) + lines.append('if [ -f $manifest ] ; then') + # Read manifest in reverse and delete files / remove empty dirs + lines.append(' sed \'1!G;h;$!d\' $manifest | while read file') + lines.append(' do') + if dryrun: + lines.append(' if [ ! -d $file ] ; then') + lines.append(' echo $file') + lines.append(' fi') + else: + lines.append(' if [ -d $file ] ; then') + # Avoid deleting a preserved directory in case it has special perms + lines.append(' if [ ! -d $preservedir/$file ] ; then') + lines.append(' rmdir $file > /dev/null 2>&1 || true') + lines.append(' fi') + lines.append(' else') + lines.append(' rm $file') + lines.append(' fi') + lines.append(' done') + if not dryrun: + lines.append(' rm $manifest') + if not deploy and not dryrun: + # May as well remove all traces + lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true') + lines.append('fi') + + if deploy: + if not nocheckspace: + # Check for available space + # FIXME This doesn't take into account files spread across multiple + # partitions, but doing that is non-trivial + # Find the part of the destination path that exists + lines.append('checkpath="$2"') + lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]') + lines.append('do') + lines.append(' checkpath=`dirname "$checkpath"`') + lines.append('done') + lines.append('freespace=`df -P $checkpath | sed "1d" | awk \'{ print $4 }\'`') + # First line of the file is the total space + lines.append('total=`head -n1 $3`') + lines.append('if [ $total -gt $freespace ] ; then') + lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') + lines.append(' exit 1') + lines.append('fi') + if not nopreserve: + # Preserve any files that exist. Note that this will add to the + # preserved list with successive deployments if the list of files + # deployed changes, but because we've deleted any previously + # deployed files at this point it will never preserve anything + # that was deployed, only files that existed prior to any deploying + # (which makes the most sense) + lines.append('cat $3 | sed "1d" | while read file fsize') + lines.append('do') + lines.append(' if [ -e $file ] ; then') + lines.append(' dest="$preservedir/$file"') + lines.append(' mkdir -p `dirname $dest`') + lines.append(' mv $file $dest') + lines.append(' fi') + lines.append('done') + lines.append('rm $3') + lines.append('mkdir -p `dirname $manifest`') + lines.append('mkdir -p $2') + if verbose: + lines.append(' tar xv -C $2 -f - | tee $manifest') + else: + lines.append(' tar xv -C $2 -f - > $manifest') + lines.append('sed -i "s!^./!$2!" $manifest') + elif not dryrun: + # Put any preserved files back + lines.append('if [ -d $preservedir ] ; then') + lines.append(' cd $preservedir') + lines.append(' find . -type f -exec mv {} /{} \;') + lines.append(' cd /') + lines.append(' rm -rf $preservedir') + lines.append('fi') + + if undeployall: + if not dryrun: + lines.append('echo "NOTE: Successfully undeployed $1"') + lines.append('done') + + # Delete the script itself + lines.append('rm $0') + lines.append('') + + return '\n'.join(lines) + + +def deploy(args, config, basepath, workspace): + """Entry point for the devtool 'deploy' subcommand""" + import re + import math + import oe.recipeutils + + check_workspace_recipe(workspace, args.recipename, checksrc=False) + + try: + host, destdir = args.target.split(':') + except ValueError: + destdir = '/' + else: + args.target = host + if not destdir.endswith('/'): + destdir += '/' + + tinfoil = setup_tinfoil(basepath=basepath) + try: + rd = oe.recipeutils.parse_recipe_simple(tinfoil.cooker, args.recipename, tinfoil.config_data) + except Exception as e: + raise DevtoolError('Exception parsing recipe %s: %s' % + (args.recipename, e)) + recipe_outdir = rd.getVar('D', True) + if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): + raise DevtoolError('No files to deploy - have you built the %s ' + 'recipe? If so, the install step has not installed ' + 'any files.' % args.recipename) + + filelist = [] + ftotalsize = 0 + for root, _, files in os.walk(recipe_outdir): + for fn in files: + # Get the size in kiB (since we'll be comparing it to the output of du -k) + # MUST use lstat() here not stat() or getfilesize() since we don't want to + # dereference symlinks + fsize = int(math.ceil(float(os.lstat(os.path.join(root, fn)).st_size)/1024)) + ftotalsize += fsize + # The path as it would appear on the target + fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) + filelist.append((fpath, fsize)) + + if args.dry_run: + print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) + for item, _ in filelist: + print(' %s' % item) + return 0 + + + extraoptions = '' + if args.no_host_check: + extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + if not args.show_status: + extraoptions += ' -q' + + # In order to delete previously deployed files and have the manifest file on + # the target, we write out a shell script and then copy it to the target + # so we can then run it (piping tar output to it). + # (We cannot use scp here, because it doesn't preserve symlinks.) + tmpdir = tempfile.mkdtemp(prefix='devtool') + try: + tmpscript = '/tmp/devtool_deploy.sh' + tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') + shellscript = _prepare_remote_script(deploy=True, + verbose=args.show_status, + nopreserve=args.no_preserve, + nocheckspace=args.no_check_space) + # Write out the script to a file + with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: + f.write(shellscript) + # Write out the file list + with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f: + f.write('%d\n' % ftotalsize) + for fpath, fsize in filelist: + f.write('%s %d\n' % (fpath, fsize)) + # Copy them to the target + ret = subprocess.call("scp %s %s/* %s:%s" % (extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + if ret != 0: + raise DevtoolError('Failed to copy script to %s - rerun with -s to ' + 'get a complete error message' % args.target) + finally: + shutil.rmtree(tmpdir) + + # Now run the script + ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s \'sh %s %s %s %s\'' % (extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) + if ret != 0: + raise DevtoolError('Deploy failed - rerun with -s to get a complete ' + 'error message') + + logger.info('Successfully deployed %s' % recipe_outdir) + + files_list = [] + for root, _, files in os.walk(recipe_outdir): + for filename in files: + filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) + files_list.append(os.path.join(destdir, filename)) + + return 0 + +def undeploy(args, config, basepath, workspace): + """Entry point for the devtool 'undeploy' subcommand""" + if args.all and args.recipename: + raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target') + elif not args.recipename and not args.all: + raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') + + extraoptions = '' + if args.no_host_check: + extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + if not args.show_status: + extraoptions += ' -q' + + args.target = args.target.split(':')[0] + + tmpdir = tempfile.mkdtemp(prefix='devtool') + try: + tmpscript = '/tmp/devtool_undeploy.sh' + shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all) + # Write out the script to a file + with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: + f.write(shellscript) + # Copy it to the target + ret = subprocess.call("scp %s %s/* %s:%s" % (extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + if ret != 0: + raise DevtoolError('Failed to copy script to %s - rerun with -s to ' + 'get a complete error message' % args.target) + finally: + shutil.rmtree(tmpdir) + + # Now run the script + ret = subprocess.call('ssh %s %s \'sh %s %s\'' % (extraoptions, args.target, tmpscript, args.recipename), shell=True) + if ret != 0: + raise DevtoolError('Undeploy failed - rerun with -s to get a complete ' + 'error message') + + if not args.all and not args.dry_run: + logger.info('Successfully undeployed %s' % args.recipename) + return 0 + + +def register_commands(subparsers, context): + """Register devtool subcommands from the deploy plugin""" + parser_deploy = subparsers.add_parser('deploy-target', + help='Deploy recipe output files to live target machine', + description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.', + group='testbuild') + parser_deploy.add_argument('recipename', help='Recipe to deploy') + parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]') + parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') + parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') + parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') + parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true') + parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true') + parser_deploy.set_defaults(func=deploy) + + parser_undeploy = subparsers.add_parser('undeploy-target', + help='Undeploy recipe output files in live target machine', + description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.', + group='testbuild') + parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') + parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname') + parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') + parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') + parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') + parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true') + parser_undeploy.set_defaults(func=undeploy) diff --git a/scripts/lib/devtool/package.py b/scripts/lib/devtool/package.py new file mode 100644 index 0000000..afb5809 --- /dev/null +++ b/scripts/lib/devtool/package.py @@ -0,0 +1,62 @@ +# Development tool - package command plugin +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool plugin containing the package subcommands""" + +import os +import subprocess +import logging +from bb.process import ExecutionError +from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, DevtoolError + +logger = logging.getLogger('devtool') + +def package(args, config, basepath, workspace): + """Entry point for the devtool 'package' subcommand""" + check_workspace_recipe(workspace, args.recipename) + + tinfoil = setup_tinfoil(basepath=basepath) + try: + tinfoil.prepare(config_only=True) + + image_pkgtype = config.get('Package', 'image_pkgtype', '') + if not image_pkgtype: + image_pkgtype = tinfoil.config_data.getVar('IMAGE_PKGTYPE', True) + + deploy_dir_pkg = tinfoil.config_data.getVar('DEPLOY_DIR_%s' % image_pkgtype.upper(), True) + finally: + tinfoil.shutdown() + + package_task = config.get('Package', 'package_task', 'package_write_%s' % image_pkgtype) + try: + exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s' % (package_task, args.recipename), watch=True) + except bb.process.ExecutionError as e: + # We've already seen the output since watch=True, so just ensure we return something to the user + return e.exitcode + + logger.info('Your packages are in %s' % deploy_dir_pkg) + + return 0 + +def register_commands(subparsers, context): + """Register devtool subcommands from the package plugin""" + if context.fixed_setup: + parser_package = subparsers.add_parser('package', + help='Build packages for a recipe', + description='Builds packages for a recipe\'s output files', + group='testbuild', order=-5) + parser_package.add_argument('recipename', help='Recipe to package') + parser_package.set_defaults(func=package) diff --git a/scripts/lib/devtool/runqemu.py b/scripts/lib/devtool/runqemu.py new file mode 100644 index 0000000..daee7fb --- /dev/null +++ b/scripts/lib/devtool/runqemu.py @@ -0,0 +1,65 @@ +# Development tool - runqemu command plugin +# +# Copyright (C) 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +"""Devtool runqemu plugin""" + +import os +import bb +import logging +import argparse +import glob +from devtool import exec_build_env_command, setup_tinfoil, DevtoolError + +logger = logging.getLogger('devtool') + +def runqemu(args, config, basepath, workspace): + """Entry point for the devtool 'runqemu' subcommand""" + + tinfoil = setup_tinfoil(config_only=True, basepath=basepath) + machine = tinfoil.config_data.getVar('MACHINE', True) + bindir_native = tinfoil.config_data.getVar('STAGING_BINDIR_NATIVE', True) + tinfoil.shutdown() + + if not glob.glob(os.path.join(bindir_native, 'qemu-system-*')): + raise DevtoolError('QEMU is not available within this SDK') + + imagename = args.imagename + if not imagename: + sdk_targets = config.get('SDK', 'sdk_targets', '').split() + if sdk_targets: + imagename = sdk_targets[0] + if not imagename: + raise DevtoolError('Unable to determine image name to run, please specify one') + + try: + exec_build_env_command(config.init_path, basepath, 'runqemu %s %s %s' % (machine, imagename, " ".join(args.args)), watch=True) + except bb.process.ExecutionError as e: + # We've already seen the output since watch=True, so just ensure we return something to the user + return e.exitcode + + return 0 + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + if context.fixed_setup: + parser_runqemu = subparsers.add_parser('runqemu', help='Run QEMU on the specified image', + description='Runs QEMU to boot the specified image', + group='testbuild', order=-20) + parser_runqemu.add_argument('imagename', help='Name of built image to boot within QEMU', nargs='?') + parser_runqemu.add_argument('args', help='Any remaining arguments are passed to the runqemu script (pass --help after imagename to see what these are)', + nargs=argparse.REMAINDER) + parser_runqemu.set_defaults(func=runqemu) diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py new file mode 100644 index 0000000..46fd12b --- /dev/null +++ b/scripts/lib/devtool/sdk.py @@ -0,0 +1,366 @@ +# Development tool - sdk-update command plugin +# +# Copyright (C) 2015-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import subprocess +import logging +import glob +import shutil +import errno +import sys +import tempfile +import re +from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError + +logger = logging.getLogger('devtool') + +def parse_locked_sigs(sigfile_path): + """Return <pn:task>:<hash> dictionary""" + sig_dict = {} + with open(sigfile_path) as f: + lines = f.readlines() + for line in lines: + if ':' in line: + taskkey, _, hashval = line.rpartition(':') + sig_dict[taskkey.strip()] = hashval.split()[0] + return sig_dict + +def generate_update_dict(sigfile_new, sigfile_old): + """Return a dict containing <pn:task>:<hash> which indicates what need to be updated""" + update_dict = {} + sigdict_new = parse_locked_sigs(sigfile_new) + sigdict_old = parse_locked_sigs(sigfile_old) + for k in sigdict_new: + if k not in sigdict_old: + update_dict[k] = sigdict_new[k] + continue + if sigdict_new[k] != sigdict_old[k]: + update_dict[k] = sigdict_new[k] + continue + return update_dict + +def get_sstate_objects(update_dict, sstate_dir): + """Return a list containing sstate objects which are to be installed""" + sstate_objects = [] + for k in update_dict: + files = set() + hashval = update_dict[k] + p = sstate_dir + '/' + hashval[:2] + '/*' + hashval + '*.tgz' + files |= set(glob.glob(p)) + p = sstate_dir + '/*/' + hashval[:2] + '/*' + hashval + '*.tgz' + files |= set(glob.glob(p)) + files = list(files) + if len(files) == 1: + sstate_objects.extend(files) + elif len(files) > 1: + logger.error("More than one matching sstate object found for %s" % hashval) + + return sstate_objects + +def mkdir(d): + try: + os.makedirs(d) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + +def install_sstate_objects(sstate_objects, src_sdk, dest_sdk): + """Install sstate objects into destination SDK""" + sstate_dir = os.path.join(dest_sdk, 'sstate-cache') + if not os.path.exists(sstate_dir): + logger.error("Missing sstate-cache directory in %s, it might not be an extensible SDK." % dest_sdk) + raise + for sb in sstate_objects: + dst = sb.replace(src_sdk, dest_sdk) + destdir = os.path.dirname(dst) + mkdir(destdir) + logger.debug("Copying %s to %s" % (sb, dst)) + shutil.copy(sb, dst) + +def check_manifest(fn, basepath): + import bb.utils + changedfiles = [] + with open(fn, 'r') as f: + for line in f: + splitline = line.split() + if len(splitline) > 1: + chksum = splitline[0] + fpath = splitline[1] + curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath)) + if chksum != curr_chksum: + logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum)) + changedfiles.append(fpath) + return changedfiles + +def sdk_update(args, config, basepath, workspace): + # Fetch locked-sigs.inc file from remote/local destination + updateserver = args.updateserver + if not updateserver: + updateserver = config.get('SDK', 'updateserver', '') + logger.debug("updateserver: %s" % updateserver) + + # Make sure we are using sdk-update from within SDK + logger.debug("basepath = %s" % basepath) + old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc') + if not os.path.exists(old_locked_sig_file_path): + logger.error("Not using devtool's sdk-update command from within an extensible SDK. Please specify correct basepath via --basepath option") + return -1 + else: + logger.debug("Found conf/locked-sigs.inc in %s" % basepath) + + if ':' in updateserver: + is_remote = True + else: + is_remote = False + + layers_dir = os.path.join(basepath, 'layers') + conf_dir = os.path.join(basepath, 'conf') + + # Grab variable values + tinfoil = setup_tinfoil(config_only=True, basepath=basepath) + try: + stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR', True) + sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS', True) + site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION', True) + finally: + tinfoil.shutdown() + + if not is_remote: + # devtool sdk-update /local/path/to/latest/sdk + new_locked_sig_file_path = os.path.join(updateserver, 'conf/locked-sigs.inc') + if not os.path.exists(new_locked_sig_file_path): + logger.error("%s doesn't exist or is not an extensible SDK" % updateserver) + return -1 + else: + logger.debug("Found conf/locked-sigs.inc in %s" % updateserver) + update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path) + logger.debug("update_dict = %s" % update_dict) + newsdk_path = updateserver + sstate_dir = os.path.join(newsdk_path, 'sstate-cache') + if not os.path.exists(sstate_dir): + logger.error("sstate-cache directory not found under %s" % newsdk_path) + return 1 + sstate_objects = get_sstate_objects(update_dict, sstate_dir) + logger.debug("sstate_objects = %s" % sstate_objects) + if len(sstate_objects) == 0: + logger.info("No need to update.") + return 0 + logger.info("Installing sstate objects into %s", basepath) + install_sstate_objects(sstate_objects, updateserver.rstrip('/'), basepath) + logger.info("Updating configuration files") + new_conf_dir = os.path.join(updateserver, 'conf') + shutil.rmtree(conf_dir) + shutil.copytree(new_conf_dir, conf_dir) + logger.info("Updating layers") + new_layers_dir = os.path.join(updateserver, 'layers') + shutil.rmtree(layers_dir) + ret = subprocess.call("cp -a %s %s" % (new_layers_dir, layers_dir), shell=True) + if ret != 0: + logger.error("Copying %s to %s failed" % (new_layers_dir, layers_dir)) + return ret + else: + # devtool sdk-update http://myhost/sdk + tmpsdk_dir = tempfile.mkdtemp() + try: + os.makedirs(os.path.join(tmpsdk_dir, 'conf')) + new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc') + # Fetch manifest from server + tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest') + ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True) + changedfiles = check_manifest(tmpmanifest, basepath) + if not changedfiles: + logger.info("Already up-to-date") + return 0 + # Update metadata + logger.debug("Updating metadata via git ...") + #Check for the status before doing a fetch and reset + if os.path.exists(os.path.join(basepath, 'layers/.git')): + out = subprocess.check_output("git status --porcelain", shell=True, cwd=layers_dir) + if not out: + ret = subprocess.call("git fetch --all; git reset --hard", shell=True, cwd=layers_dir) + else: + logger.error("Failed to update metadata as there have been changes made to it. Aborting."); + logger.error("Changed files:\n%s" % out); + return -1 + else: + ret = -1 + if ret != 0: + ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir) + if ret != 0: + logger.error("Updating metadata via git failed") + return ret + logger.debug("Updating conf files ...") + for changedfile in changedfiles: + ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir) + if ret != 0: + logger.error("Updating %s failed" % changedfile) + return ret + + # Check if UNINATIVE_CHECKSUM changed + uninative = False + if 'conf/local.conf' in changedfiles: + def read_uninative_checksums(fn): + chksumitems = [] + with open(fn, 'r') as f: + for line in f: + if line.startswith('UNINATIVE_CHECKSUM'): + splitline = re.split(r'[\[\]"\']', line) + if len(splitline) > 3: + chksumitems.append((splitline[1], splitline[3])) + return chksumitems + + oldsums = read_uninative_checksums(os.path.join(basepath, 'conf/local.conf')) + newsums = read_uninative_checksums(os.path.join(tmpsdk_dir, 'conf/local.conf')) + if oldsums != newsums: + uninative = True + for buildarch, chksum in newsums: + uninative_file = os.path.join('downloads', 'uninative', chksum, '%s-nativesdk-libc.tar.bz2' % buildarch) + mkdir(os.path.join(tmpsdk_dir, os.path.dirname(uninative_file))) + ret = subprocess.call("wget -q -O %s %s/%s" % (uninative_file, updateserver, uninative_file), shell=True, cwd=tmpsdk_dir) + + # Ok, all is well at this point - move everything over + tmplayers_dir = os.path.join(tmpsdk_dir, 'layers') + if os.path.exists(tmplayers_dir): + shutil.rmtree(layers_dir) + shutil.move(tmplayers_dir, layers_dir) + for changedfile in changedfiles: + destfile = os.path.join(basepath, changedfile) + os.remove(destfile) + shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile) + os.remove(os.path.join(conf_dir, 'sdk-conf-manifest')) + shutil.move(tmpmanifest, conf_dir) + if uninative: + shutil.rmtree(os.path.join(basepath, 'downloads', 'uninative')) + shutil.move(os.path.join(tmpsdk_dir, 'downloads', 'uninative'), os.path.join(basepath, 'downloads')) + + if not sstate_mirrors: + with open(os.path.join(conf_dir, 'site.conf'), 'a') as f: + f.write('SCONF_VERSION = "%s"\n' % site_conf_version) + f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver) + finally: + shutil.rmtree(tmpsdk_dir) + + if not args.skip_prepare: + # Find all potentially updateable tasks + sdk_update_targets = [] + tasks = ['do_populate_sysroot', 'do_packagedata'] + for root, _, files in os.walk(stamps_dir): + for fn in files: + if not '.sigdata.' in fn: + for task in tasks: + if '.%s.' % task in fn or '.%s_setscene.' % task in fn: + sdk_update_targets.append('%s:%s' % (os.path.basename(root), task)) + # Run bitbake command for the whole SDK + logger.info("Preparing build system... (This may take some time.)") + try: + exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT) + output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT) + runlines = [] + for line in output.splitlines(): + if 'Running task ' in line: + runlines.append(line) + if runlines: + logger.error('Unexecuted tasks found in preparation log:\n %s' % '\n '.join(runlines)) + return -1 + except bb.process.ExecutionError as e: + logger.error('Preparation failed:\n%s' % e.stdout) + return -1 + return 0 + +def sdk_install(args, config, basepath, workspace): + """Entry point for the devtool sdk-install command""" + + import oe.recipeutils + import bb.process + + for recipe in args.recipename: + if recipe in workspace: + raise DevtoolError('recipe %s is a recipe in your workspace' % recipe) + + tasks = ['do_populate_sysroot', 'do_packagedata'] + stampprefixes = {} + def checkstamp(recipe): + stampprefix = stampprefixes[recipe] + stamps = glob.glob(stampprefix + '*') + for stamp in stamps: + if '.sigdata.' not in stamp and stamp.startswith((stampprefix + '.', stampprefix + '_setscene.')): + return True + else: + return False + + install_recipes = [] + tinfoil = setup_tinfoil(config_only=False, basepath=basepath) + try: + for recipe in args.recipename: + rd = parse_recipe(config, tinfoil, recipe, True) + if not rd: + return 1 + stampprefixes[recipe] = '%s.%s' % (rd.getVar('STAMP', True), tasks[0]) + if checkstamp(recipe): + logger.info('%s is already installed' % recipe) + else: + install_recipes.append(recipe) + finally: + tinfoil.shutdown() + + if install_recipes: + logger.info('Installing %s...' % ', '.join(install_recipes)) + install_tasks = [] + for recipe in install_recipes: + for task in tasks: + if recipe.endswith('-native') and 'package' in task: + continue + install_tasks.append('%s:%s' % (recipe, task)) + options = '' + if not args.allow_build: + options += ' --setscene-only' + try: + exec_build_env_command(config.init_path, basepath, 'bitbake %s %s' % (options, ' '.join(install_tasks)), watch=True) + except bb.process.ExecutionError as e: + raise DevtoolError('Failed to install %s:\n%s' % (recipe, str(e))) + failed = False + for recipe in install_recipes: + if checkstamp(recipe): + logger.info('Successfully installed %s' % recipe) + else: + raise DevtoolError('Failed to install %s - unavailable' % recipe) + failed = True + if failed: + return 2 + +def register_commands(subparsers, context): + """Register devtool subcommands from the sdk plugin""" + if context.fixed_setup: + parser_sdk = subparsers.add_parser('sdk-update', + help='Update SDK components', + description='Updates installed SDK components from a remote server', + group='sdk') + updateserver = context.config.get('SDK', 'updateserver', '') + if updateserver: + parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from (default %s)' % updateserver, nargs='?') + else: + parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from') + parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)') + parser_sdk.set_defaults(func=sdk_update) + + parser_sdk_install = subparsers.add_parser('sdk-install', + help='Install additional SDK components', + description='Installs additional recipe development files into the SDK. (You can use "devtool search" to find available recipes.)', + group='sdk') + parser_sdk_install.add_argument('recipename', help='Name of the recipe to install the development artifacts for', nargs='+') + parser_sdk_install.add_argument('-s', '--allow-build', help='Allow building requested item(s) from source', action='store_true') + parser_sdk_install.set_defaults(func=sdk_install) diff --git a/scripts/lib/devtool/search.py b/scripts/lib/devtool/search.py new file mode 100644 index 0000000..b44bed7 --- /dev/null +++ b/scripts/lib/devtool/search.py @@ -0,0 +1,88 @@ +# Development tool - search command plugin +# +# Copyright (C) 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +"""Devtool search plugin""" + +import os +import bb +import logging +import argparse +import re +from devtool import setup_tinfoil, parse_recipe, DevtoolError + +logger = logging.getLogger('devtool') + +def search(args, config, basepath, workspace): + """Entry point for the devtool 'search' subcommand""" + + tinfoil = setup_tinfoil(config_only=False, basepath=basepath) + try: + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) + defsummary = tinfoil.config_data.getVar('SUMMARY', False) or '' + + keyword_rc = re.compile(args.keyword) + + for fn in os.listdir(pkgdata_dir): + pfn = os.path.join(pkgdata_dir, fn) + if not os.path.isfile(pfn): + continue + + packages = [] + match = False + if keyword_rc.search(fn): + match = True + + if not match: + with open(pfn, 'r') as f: + for line in f: + if line.startswith('PACKAGES:'): + packages = line.split(':', 1)[1].strip().split() + + for pkg in packages: + if keyword_rc.search(pkg): + match = True + break + if os.path.exists(os.path.join(pkgdata_dir, 'runtime', pkg + '.packaged')): + with open(os.path.join(pkgdata_dir, 'runtime', pkg), 'r') as f: + for line in f: + if ': ' in line: + splitline = line.split(':', 1) + key = splitline[0] + value = splitline[1].strip() + if key in ['PKG_%s' % pkg, 'DESCRIPTION', 'FILES_INFO'] or key.startswith('FILERPROVIDES_'): + if keyword_rc.search(value): + match = True + break + + if match: + rd = parse_recipe(config, tinfoil, fn, True) + summary = rd.getVar('SUMMARY', True) + if summary == rd.expand(defsummary): + summary = '' + print("%s %s" % (fn.ljust(20), summary)) + finally: + tinfoil.shutdown() + + return 0 + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + parser_search = subparsers.add_parser('search', help='Search available recipes', + description='Searches for available target recipes. Matches on recipe name, package name, description and installed files, and prints the recipe name on match.', + group='info') + parser_search.add_argument('keyword', help='Keyword to search for (regular expression syntax allowed)') + parser_search.set_defaults(func=search, no_workspace=True) diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py new file mode 100644 index 0000000..f414860 --- /dev/null +++ b/scripts/lib/devtool/standard.py @@ -0,0 +1,1452 @@ +# Development tool - standard commands plugin +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool standard plugins""" + +import os +import sys +import re +import shutil +import subprocess +import tempfile +import logging +import argparse +import argparse_oe +import scriptutils +import errno +import glob +from collections import OrderedDict +from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, DevtoolError +from devtool import parse_recipe + +logger = logging.getLogger('devtool') + + +def add(args, config, basepath, workspace): + """Entry point for the devtool 'add' subcommand""" + import bb + import oe.recipeutils + + if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: + raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') + + # These are positional arguments, but because we're nice, allow + # specifying e.g. source tree without name, or fetch URI without name or + # source tree (if we can detect that that is what the user meant) + if '://' in args.recipename: + if not args.fetchuri: + if args.fetch: + raise DevtoolError('URI specified as positional argument as well as -f/--fetch') + args.fetchuri = args.recipename + args.recipename = '' + elif args.srctree and '://' in args.srctree: + if not args.fetchuri: + if args.fetch: + raise DevtoolError('URI specified as positional argument as well as -f/--fetch') + args.fetchuri = args.srctree + args.srctree = '' + elif args.recipename and not args.srctree: + if os.sep in args.recipename: + args.srctree = args.recipename + args.recipename = None + elif os.path.isdir(args.recipename): + logger.warn('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) + + if args.fetch: + if args.fetchuri: + raise DevtoolError('URI specified as positional argument as well as -f/--fetch') + else: + # FIXME should show a warning that -f/--fetch is deprecated here + args.fetchuri = args.fetch + + if args.recipename: + if args.recipename in workspace: + raise DevtoolError("recipe %s is already in your workspace" % + args.recipename) + reason = oe.recipeutils.validate_pn(args.recipename) + if reason: + raise DevtoolError(reason) + + # FIXME this ought to be in validate_pn but we're using that in other contexts + if '/' in args.recipename: + raise DevtoolError('"/" is not a valid character in recipe names') + + if args.srctree: + srctree = os.path.abspath(args.srctree) + srctreeparent = None + tmpsrcdir = None + else: + srctree = None + srctreeparent = get_default_srctree(config) + bb.utils.mkdirhier(srctreeparent) + tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) + + if srctree and os.path.exists(srctree): + if args.fetchuri: + if not os.path.isdir(srctree): + raise DevtoolError("Cannot fetch into source tree path %s as " + "it exists and is not a directory" % + srctree) + elif os.listdir(srctree): + raise DevtoolError("Cannot fetch into source tree path %s as " + "it already exists and is non-empty" % + srctree) + elif not args.fetchuri: + if args.srctree: + raise DevtoolError("Specified source tree %s could not be found" % + args.srctree) + elif srctree: + raise DevtoolError("No source tree exists at default path %s - " + "either create and populate this directory, " + "or specify a path to a source tree, or a " + "URI to fetch source from" % srctree) + else: + raise DevtoolError("You must either specify a source tree " + "or a URI to fetch source from") + + if args.version: + if '_' in args.version or ' ' in args.version: + raise DevtoolError('Invalid version string "%s"' % args.version) + + if args.color == 'auto' and sys.stdout.isatty(): + color = 'always' + else: + color = args.color + extracmdopts = '' + if args.fetchuri: + source = args.fetchuri + if srctree: + extracmdopts += ' -x %s' % srctree + else: + extracmdopts += ' -x %s' % tmpsrcdir + else: + source = srctree + if args.recipename: + extracmdopts += ' -N %s' % args.recipename + if args.version: + extracmdopts += ' -V %s' % args.version + if args.binary: + extracmdopts += ' -b' + if args.also_native: + extracmdopts += ' --also-native' + if args.src_subdir: + extracmdopts += ' --src-subdir "%s"' % args.src_subdir + + tempdir = tempfile.mkdtemp(prefix='devtool') + try: + try: + stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, tempdir, source, extracmdopts)) + except bb.process.ExecutionError as e: + if e.exitcode == 15: + raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') + else: + raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) + + recipes = glob.glob(os.path.join(tempdir, '*.bb')) + if recipes: + recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] + if recipename in workspace: + raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) + recipedir = os.path.join(config.workspace_path, 'recipes', recipename) + bb.utils.mkdirhier(recipedir) + recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) + appendfile = recipe_to_append(recipefile, config) + if os.path.exists(appendfile): + # This shouldn't be possible, but just in case + raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') + if os.path.exists(recipefile): + raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) + if tmpsrcdir: + srctree = os.path.join(srctreeparent, recipename) + if os.path.exists(tmpsrcdir): + if os.path.exists(srctree): + if os.path.isdir(srctree): + try: + os.rmdir(srctree) + except OSError as e: + if e.errno == errno.ENOTEMPTY: + raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) + else: + raise + else: + raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) + logger.info('Using default source tree path %s' % srctree) + shutil.move(tmpsrcdir, srctree) + else: + raise DevtoolError('Couldn\'t find source tree created by recipetool') + bb.utils.mkdirhier(recipedir) + shutil.move(recipes[0], recipefile) + # Move any additional files created by recipetool + for fn in os.listdir(tempdir): + shutil.move(os.path.join(tempdir, fn), recipedir) + else: + raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) + attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) + if os.path.exists(attic_recipe): + logger.warn('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) + finally: + if tmpsrcdir and os.path.exists(tmpsrcdir): + shutil.rmtree(tmpsrcdir) + shutil.rmtree(tempdir) + + for fn in os.listdir(recipedir): + _add_md5(config, recipename, os.path.join(recipedir, fn)) + + if args.fetchuri and not args.no_git: + setup_git_repo(srctree, args.version, 'devtool') + + initial_rev = None + if os.path.exists(os.path.join(srctree, '.git')): + (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) + initial_rev = stdout.rstrip() + + tinfoil = setup_tinfoil(config_only=True, basepath=basepath) + rd = oe.recipeutils.parse_recipe(recipefile, None, tinfoil.config_data) + if not rd: + return 1 + + if args.src_subdir: + srctree = os.path.join(srctree, args.src_subdir) + + bb.utils.mkdirhier(os.path.dirname(appendfile)) + with open(appendfile, 'w') as f: + f.write('inherit externalsrc\n') + f.write('EXTERNALSRC = "%s"\n' % srctree) + + b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) + if b_is_s: + f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) + if initial_rev: + f.write('\n# initial_rev: %s\n' % initial_rev) + + if args.binary: + f.write('do_install_append() {\n') + f.write(' rm -rf ${D}/.git\n') + f.write(' rm -f ${D}/singletask.lock\n') + f.write('}\n') + + if bb.data.inherits_class('npm', rd): + f.write('do_install_append() {\n') + f.write(' # Remove files added to source dir by devtool/externalsrc\n') + f.write(' rm -f ${NPM_INSTALLDIR}/singletask.lock\n') + f.write(' rm -rf ${NPM_INSTALLDIR}/.git\n') + f.write(' rm -rf ${NPM_INSTALLDIR}/oe-local-files\n') + f.write(' for symlink in ${EXTERNALSRC_SYMLINKS} ; do\n') + f.write(' rm -f ${NPM_INSTALLDIR}/${symlink%%:*}\n') + f.write(' done\n') + f.write('}\n') + + _add_md5(config, recipename, appendfile) + + logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) + + tinfoil.shutdown() + + return 0 + + +def _check_compatible_recipe(pn, d): + """Check if the recipe is supported by devtool""" + if pn == 'perf': + raise DevtoolError("The perf recipe does not actually check out " + "source and thus cannot be supported by this tool") + + if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): + raise DevtoolError("The %s recipe is not supported by this tool" % pn) + + if bb.data.inherits_class('image', d): + raise DevtoolError("The %s recipe is an image, and therefore is not " + "supported by this tool" % pn) + + if bb.data.inherits_class('populate_sdk', d): + raise DevtoolError("The %s recipe is an SDK, and therefore is not " + "supported by this tool" % pn) + + if bb.data.inherits_class('packagegroup', d): + raise DevtoolError("The %s recipe is a packagegroup, and therefore is " + "not supported by this tool" % pn) + + if bb.data.inherits_class('meta', d): + raise DevtoolError("The %s recipe is a meta-recipe, and therefore is " + "not supported by this tool" % pn) + + if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC', True): + raise DevtoolError("externalsrc is currently enabled for the %s " + "recipe. This prevents the normal do_patch task " + "from working. You will need to disable this " + "first." % pn) + +def _move_file(src, dst): + """Move a file. Creates all the directory components of destination path.""" + dst_d = os.path.dirname(dst) + if dst_d: + bb.utils.mkdirhier(dst_d) + shutil.move(src, dst) + +def _git_ls_tree(repodir, treeish='HEAD', recursive=False): + """List contents of a git treeish""" + import bb + cmd = ['git', 'ls-tree', '-z', treeish] + if recursive: + cmd.append('-r') + out, _ = bb.process.run(cmd, cwd=repodir) + ret = {} + for line in out.split('\0'): + if line: + split = line.split(None, 4) + ret[split[3]] = split[0:3] + return ret + +def _git_exclude_path(srctree, path): + """Return pathspec (list of paths) that excludes certain path""" + # NOTE: "Filtering out" files/paths in this way is not entirely reliable - + # we don't catch files that are deleted, for example. A more reliable way + # to implement this would be to use "negative pathspecs" which were + # introduced in Git v1.9.0. Revisit this when/if the required Git version + # becomes greater than that. + path = os.path.normpath(path) + recurse = True if len(path.split(os.path.sep)) > 1 else False + git_files = _git_ls_tree(srctree, 'HEAD', recurse).keys() + if path in git_files: + git_files.remove(path) + return git_files + else: + return ['.'] + +def _ls_tree(directory): + """Recursive listing of files in a directory""" + ret = [] + for root, dirs, files in os.walk(directory): + ret.extend([os.path.relpath(os.path.join(root, fname), directory) for + fname in files]) + return ret + + +def extract(args, config, basepath, workspace): + """Entry point for the devtool 'extract' subcommand""" + import bb + + tinfoil = _prep_extract_operation(config, basepath, args.recipename) + if not tinfoil: + # Error already shown + return 1 + + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + + srctree = os.path.abspath(args.srctree) + initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd) + logger.info('Source tree extracted to %s' % srctree) + + if initial_rev: + return 0 + else: + return 1 + +def sync(args, config, basepath, workspace): + """Entry point for the devtool 'sync' subcommand""" + import bb + + tinfoil = _prep_extract_operation(config, basepath, args.recipename) + if not tinfoil: + # Error already shown + return 1 + + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + + srctree = os.path.abspath(args.srctree) + initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd) + logger.info('Source tree %s synchronized' % srctree) + + if initial_rev: + return 0 + else: + return 1 + +class BbTaskExecutor(object): + """Class for executing bitbake tasks for a recipe + + FIXME: This is very awkward. Unfortunately it's not currently easy to + properly execute tasks outside of bitbake itself, until then this has to + suffice if we are to handle e.g. linux-yocto's extra tasks + """ + + def __init__(self, rdata): + self.rdata = rdata + self.executed = [] + + def exec_func(self, func, report): + """Run bitbake task function""" + if not func in self.executed: + deps = self.rdata.getVarFlag(func, 'deps', False) + if deps: + for taskdepfunc in deps: + self.exec_func(taskdepfunc, True) + if report: + logger.info('Executing %s...' % func) + fn = self.rdata.getVar('FILE', True) + localdata = bb.build._task_data(fn, func, self.rdata) + try: + bb.build.exec_func(func, localdata) + except bb.build.FuncFailed as e: + raise DevtoolError(str(e)) + self.executed.append(func) + + +class PatchTaskExecutor(BbTaskExecutor): + def __init__(self, rdata): + self.check_git = False + super(PatchTaskExecutor, self).__init__(rdata) + + def exec_func(self, func, report): + from oe.patch import GitApplyTree + srcsubdir = self.rdata.getVar('S', True) + haspatches = False + if func == 'do_patch': + patchdir = os.path.join(srcsubdir, 'patches') + if os.path.exists(patchdir): + if os.listdir(patchdir): + haspatches = True + else: + os.rmdir(patchdir) + + super(PatchTaskExecutor, self).exec_func(func, report) + if self.check_git and os.path.exists(srcsubdir): + if func == 'do_patch': + if os.path.exists(patchdir): + shutil.rmtree(patchdir) + if haspatches: + stdout, _ = bb.process.run('git status --porcelain patches', cwd=srcsubdir) + if stdout: + bb.process.run('git checkout patches', cwd=srcsubdir) + + stdout, _ = bb.process.run('git status --porcelain', cwd=srcsubdir) + if stdout: + bb.process.run('git add .; git commit -a -m "Committing changes from %s\n\n%s"' % (func, GitApplyTree.ignore_commit_prefix + ' - from %s' % func), cwd=srcsubdir) + + +def _prep_extract_operation(config, basepath, recipename, tinfoil=None): + """HACK: Ugly workaround for making sure that requirements are met when + trying to extract a package. Returns the tinfoil instance to be used.""" + if not tinfoil: + tinfoil = setup_tinfoil(basepath=basepath) + + rd = parse_recipe(config, tinfoil, recipename, True) + if not rd: + return None + + if bb.data.inherits_class('kernel-yocto', rd): + tinfoil.shutdown() + try: + stdout, _ = exec_build_env_command(config.init_path, basepath, + 'bitbake kern-tools-native') + tinfoil = setup_tinfoil(basepath=basepath) + except bb.process.ExecutionError as err: + raise DevtoolError("Failed to build kern-tools-native:\n%s" % + err.stdout) + return tinfoil + + +def _extract_source(srctree, keep_temp, devbranch, sync, d): + """Extract sources of a recipe""" + import bb.event + import oe.recipeutils + + def eventfilter(name, handler, event, d): + """Bitbake event filter for devtool extract operation""" + if name == 'base_eventhandler': + return True + else: + return False + + if hasattr(bb.event, 'set_eventfilter'): + bb.event.set_eventfilter(eventfilter) + + pn = d.getVar('PN', True) + + _check_compatible_recipe(pn, d) + + if sync: + if not os.path.exists(srctree): + raise DevtoolError("output path %s does not exist" % srctree) + else: + if os.path.exists(srctree): + if not os.path.isdir(srctree): + raise DevtoolError("output path %s exists and is not a directory" % + srctree) + elif os.listdir(srctree): + raise DevtoolError("output path %s already exists and is " + "non-empty" % srctree) + + if 'noexec' in (d.getVarFlags('do_unpack', False) or []): + raise DevtoolError("The %s recipe has do_unpack disabled, unable to " + "extract source" % pn) + + if not sync: + # Prepare for shutil.move later on + bb.utils.mkdirhier(srctree) + os.rmdir(srctree) + + # We don't want notes to be printed, they are too verbose + origlevel = bb.logger.getEffectiveLevel() + if logger.getEffectiveLevel() > logging.DEBUG: + bb.logger.setLevel(logging.WARNING) + + initial_rev = None + tempdir = tempfile.mkdtemp(prefix='devtool') + try: + crd = d.createCopy() + # Make a subdir so we guard against WORKDIR==S + workdir = os.path.join(tempdir, 'workdir') + crd.setVar('WORKDIR', workdir) + crd.setVar('T', os.path.join(tempdir, 'temp')) + if not crd.getVar('S', True).startswith(workdir): + # Usually a shared workdir recipe (kernel, gcc) + # Try to set a reasonable default + if bb.data.inherits_class('kernel', d): + crd.setVar('S', '${WORKDIR}/source') + else: + crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S', True))) + if bb.data.inherits_class('kernel', d): + # We don't want to move the source to STAGING_KERNEL_DIR here + crd.setVar('STAGING_KERNEL_DIR', '${S}') + + task_executor = PatchTaskExecutor(crd) + + crd.setVar('EXTERNALSRC_forcevariable', '') + + logger.info('Fetching %s...' % pn) + task_executor.exec_func('do_fetch', False) + logger.info('Unpacking...') + task_executor.exec_func('do_unpack', False) + if bb.data.inherits_class('kernel-yocto', d): + # Extra step for kernel to populate the source directory + logger.info('Doing kernel checkout...') + task_executor.exec_func('do_kernel_checkout', False) + srcsubdir = crd.getVar('S', True) + + task_executor.check_git = True + + # Move local source files into separate subdir + recipe_patches = [os.path.basename(patch) for patch in + oe.recipeutils.get_recipe_patches(crd)] + local_files = oe.recipeutils.get_recipe_local_files(crd) + local_files = [fname for fname in local_files if + os.path.exists(os.path.join(workdir, fname))] + if local_files: + for fname in local_files: + _move_file(os.path.join(workdir, fname), + os.path.join(tempdir, 'oe-local-files', fname)) + with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'), + 'w') as f: + f.write('# Ignore local files, by default. Remove this file ' + 'if you want to commit the directory to Git\n*\n') + + if srcsubdir == workdir: + # Find non-patch non-local sources that were "unpacked" to srctree + # directory + src_files = [fname for fname in _ls_tree(workdir) if + os.path.basename(fname) not in recipe_patches] + # Force separate S so that patch files can be left out from srctree + srcsubdir = tempfile.mkdtemp(dir=workdir) + crd.setVar('S', srcsubdir) + # Move source files to S + for path in src_files: + _move_file(os.path.join(workdir, path), + os.path.join(srcsubdir, path)) + elif os.path.dirname(srcsubdir) != workdir: + # Handle if S is set to a subdirectory of the source + srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0]) + + scriptutils.git_convert_standalone_clone(srcsubdir) + + # Make sure that srcsubdir exists + bb.utils.mkdirhier(srcsubdir) + if not os.path.exists(srcsubdir) or not os.listdir(srcsubdir): + logger.warning("no source unpacked to S, either the %s recipe " + "doesn't use any source or the correct source " + "directory could not be determined" % pn) + + setup_git_repo(srcsubdir, crd.getVar('PV', True), devbranch) + + (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) + initial_rev = stdout.rstrip() + + crd.setVar('PATCHTOOL', 'git') + + logger.info('Patching...') + task_executor.exec_func('do_patch', False) + + bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) + + kconfig = None + if bb.data.inherits_class('kernel-yocto', d): + # Store generate and store kernel config + logger.info('Generating kernel config') + task_executor.exec_func('do_configure', False) + kconfig = os.path.join(crd.getVar('B', True), '.config') + + + tempdir_localdir = os.path.join(tempdir, 'oe-local-files') + srctree_localdir = os.path.join(srctree, 'oe-local-files') + + if sync: + bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) + + # Move oe-local-files directory to srctree + # As the oe-local-files is not part of the constructed git tree, + # remove them directly during the synchrounizating might surprise + # the users. Instead, we move it to oe-local-files.bak and remind + # user in the log message. + if os.path.exists(srctree_localdir + '.bak'): + shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') + + if os.path.exists(srctree_localdir): + logger.info('Backing up current local file directory %s' % srctree_localdir) + shutil.move(srctree_localdir, srctree_localdir + '.bak') + + if os.path.exists(tempdir_localdir): + logger.info('Syncing local source files to srctree...') + shutil.copytree(tempdir_localdir, srctree_localdir) + else: + # Move oe-local-files directory to srctree + if os.path.exists(tempdir_localdir): + logger.info('Adding local source files to srctree...') + shutil.move(tempdir_localdir, srcsubdir) + + shutil.move(srcsubdir, srctree) + + if kconfig: + logger.info('Copying kernel config to srctree') + shutil.copy2(kconfig, srctree) + + finally: + bb.logger.setLevel(origlevel) + + if keep_temp: + logger.info('Preserving temporary directory %s' % tempdir) + else: + shutil.rmtree(tempdir) + return initial_rev + +def _add_md5(config, recipename, filename): + """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" + import bb.utils + + def addfile(fn): + md5 = bb.utils.md5_file(fn) + with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a') as f: + f.write('%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5)) + + if os.path.isdir(filename): + for root, _, files in os.walk(filename): + for f in files: + addfile(os.path.join(root, f)) + else: + addfile(filename) + +def _check_preserve(config, recipename): + """Check if a file was manually changed and needs to be saved in 'attic' + directory""" + import bb.utils + origfile = os.path.join(config.workspace_path, '.devtool_md5') + newfile = os.path.join(config.workspace_path, '.devtool_md5_new') + preservepath = os.path.join(config.workspace_path, 'attic', recipename) + with open(origfile, 'r') as f: + with open(newfile, 'w') as tf: + for line in f.readlines(): + splitline = line.rstrip().split('|') + if splitline[0] == recipename: + removefile = os.path.join(config.workspace_path, splitline[1]) + try: + md5 = bb.utils.md5_file(removefile) + except IOError as err: + if err.errno == 2: + # File no longer exists, skip it + continue + else: + raise + if splitline[2] != md5: + bb.utils.mkdirhier(preservepath) + preservefile = os.path.basename(removefile) + logger.warn('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) + shutil.move(removefile, os.path.join(preservepath, preservefile)) + else: + os.remove(removefile) + else: + tf.write(line) + os.rename(newfile, origfile) + +def modify(args, config, basepath, workspace): + """Entry point for the devtool 'modify' subcommand""" + import bb + import oe.recipeutils + + if args.recipename in workspace: + raise DevtoolError("recipe %s is already in your workspace" % + args.recipename) + + tinfoil = setup_tinfoil(basepath=basepath) + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + + pn = rd.getVar('PN', True) + if pn != args.recipename: + logger.info('Mapping %s to %s' % (args.recipename, pn)) + if pn in workspace: + raise DevtoolError("recipe %s is already in your workspace" % + pn) + + if args.srctree: + srctree = os.path.abspath(args.srctree) + else: + srctree = get_default_srctree(config, pn) + + if args.no_extract and not os.path.isdir(srctree): + raise DevtoolError("--no-extract specified and source path %s does " + "not exist or is not a directory" % + srctree) + if not args.no_extract: + tinfoil = _prep_extract_operation(config, basepath, pn, tinfoil) + if not tinfoil: + # Error already shown + return 1 + + recipefile = rd.getVar('FILE', True) + appendfile = recipe_to_append(recipefile, config, args.wildcard) + if os.path.exists(appendfile): + raise DevtoolError("Another variant of recipe %s is already in your " + "workspace (only one variant of a recipe can " + "currently be worked on at once)" + % pn) + + _check_compatible_recipe(pn, rd) + + initial_rev = None + commits = [] + if not args.no_extract: + initial_rev = _extract_source(srctree, False, args.branch, False, rd) + if not initial_rev: + return 1 + logger.info('Source tree extracted to %s' % srctree) + # Get list of commits since this revision + (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) + commits = stdout.split() + else: + if os.path.exists(os.path.join(srctree, '.git')): + # Check if it's a tree previously extracted by us + try: + (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) + except bb.process.ExecutionError: + stdout = '' + for line in stdout.splitlines(): + if line.startswith('*'): + (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) + initial_rev = stdout.rstrip() + if not initial_rev: + # Otherwise, just grab the head revision + (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) + initial_rev = stdout.rstrip() + + # Check that recipe isn't using a shared workdir + s = os.path.abspath(rd.getVar('S', True)) + workdir = os.path.abspath(rd.getVar('WORKDIR', True)) + if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: + # Handle if S is set to a subdirectory of the source + srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] + srctree = os.path.join(srctree, srcsubdir) + + bb.utils.mkdirhier(os.path.dirname(appendfile)) + with open(appendfile, 'w') as f: + f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n') + # Local files can be modified/tracked in separate subdir under srctree + # Mostly useful for packages with S != WORKDIR + f.write('FILESPATH_prepend := "%s:"\n' % + os.path.join(srctree, 'oe-local-files')) + + f.write('\ninherit externalsrc\n') + f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') + f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) + + b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) + if b_is_s: + f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) + + if bb.data.inherits_class('kernel', rd): + f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' + 'do_fetch do_unpack do_patch do_kernel_configme do_kernel_configcheck"\n') + f.write('\ndo_configure_append() {\n' + ' cp ${B}/.config ${S}/.config.baseline\n' + ' ln -sfT ${B}/.config ${S}/.config.new\n' + '}\n') + if initial_rev: + f.write('\n# initial_rev: %s\n' % initial_rev) + for commit in commits: + f.write('# commit: %s\n' % commit) + + _add_md5(config, pn, appendfile) + + logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) + + tinfoil.shutdown() + + return 0 + +def _get_patchset_revs(srctree, recipe_path, initial_rev=None): + """Get initial and update rev of a recipe. These are the start point of the + whole patchset and start point for the patches to be re-generated/updated. + """ + import bb + + # Parse initial rev from recipe if not specified + commits = [] + with open(recipe_path, 'r') as f: + for line in f: + if line.startswith('# initial_rev:'): + if not initial_rev: + initial_rev = line.split(':')[-1].strip() + elif line.startswith('# commit:'): + commits.append(line.split(':')[-1].strip()) + + update_rev = initial_rev + changed_revs = None + if initial_rev: + # Find first actually changed revision + stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % + initial_rev, cwd=srctree) + newcommits = stdout.split() + for i in xrange(min(len(commits), len(newcommits))): + if newcommits[i] == commits[i]: + update_rev = commits[i] + + try: + stdout, _ = bb.process.run('git cherry devtool-patched', + cwd=srctree) + except bb.process.ExecutionError as err: + stdout = None + + if stdout is not None: + changed_revs = [] + for line in stdout.splitlines(): + if line.startswith('+ '): + rev = line.split()[1] + if rev in newcommits: + changed_revs.append(rev) + + return initial_rev, update_rev, changed_revs + +def _remove_file_entries(srcuri, filelist): + """Remove file:// entries from SRC_URI""" + remaining = filelist[:] + entries = [] + for fname in filelist: + basename = os.path.basename(fname) + for i in xrange(len(srcuri)): + if (srcuri[i].startswith('file://') and + os.path.basename(srcuri[i].split(';')[0]) == basename): + entries.append(srcuri[i]) + remaining.remove(fname) + srcuri.pop(i) + break + return entries, remaining + +def _remove_source_files(args, files, destpath): + """Unlink existing patch files""" + for path in files: + if args.append: + if not destpath: + raise Exception('destpath should be set here') + path = os.path.join(destpath, os.path.basename(path)) + + if os.path.exists(path): + logger.info('Removing file %s' % path) + # FIXME "git rm" here would be nice if the file in question is + # tracked + # FIXME there's a chance that this file is referred to by + # another recipe, in which case deleting wouldn't be the + # right thing to do + os.remove(path) + # Remove directory if empty + try: + os.rmdir(os.path.dirname(path)) + except OSError as ose: + if ose.errno != errno.ENOTEMPTY: + raise + + +def _export_patches(srctree, rd, start_rev, destdir): + """Export patches from srctree to given location. + Returns three-tuple of dicts: + 1. updated - patches that already exist in SRCURI + 2. added - new patches that don't exist in SRCURI + 3 removed - patches that exist in SRCURI but not in exported patches + In each dict the key is the 'basepath' of the URI and value is the + absolute path to the existing file in recipe space (if any). + """ + import oe.recipeutils + from oe.patch import GitApplyTree + updated = OrderedDict() + added = OrderedDict() + seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') + + existing_patches = dict((os.path.basename(path), path) for path in + oe.recipeutils.get_recipe_patches(rd)) + + # Generate patches from Git, exclude local files directory + patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') + GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec) + + new_patches = sorted(os.listdir(destdir)) + for new_patch in new_patches: + # Strip numbering from patch names. If it's a git sequence named patch, + # the numbers might not match up since we are starting from a different + # revision This does assume that people are using unique shortlog + # values, but they ought to be anyway... + new_basename = seqpatch_re.match(new_patch).group(2) + found = False + for old_patch in existing_patches: + old_basename = seqpatch_re.match(old_patch).group(2) + if new_basename == old_basename: + updated[new_patch] = existing_patches.pop(old_patch) + found = True + # Rename patch files + if new_patch != old_patch: + os.rename(os.path.join(destdir, new_patch), + os.path.join(destdir, old_patch)) + break + if not found: + added[new_patch] = None + return (updated, added, existing_patches) + + +def _create_kconfig_diff(srctree, rd, outfile): + """Create a kconfig fragment""" + # Only update config fragment if both config files exist + orig_config = os.path.join(srctree, '.config.baseline') + new_config = os.path.join(srctree, '.config.new') + if os.path.exists(orig_config) and os.path.exists(new_config): + cmd = ['diff', '--new-line-format=%L', '--old-line-format=', + '--unchanged-line-format=', orig_config, new_config] + pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = pipe.communicate() + if pipe.returncode == 1: + logger.info("Updating config fragment %s" % outfile) + with open(outfile, 'w') as fobj: + fobj.write(stdout) + elif pipe.returncode == 0: + logger.info("Would remove config fragment %s" % outfile) + if os.path.exists(outfile): + # Remove fragment file in case of empty diff + logger.info("Removing config fragment %s" % outfile) + os.unlink(outfile) + else: + raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) + return True + return False + + +def _export_local_files(srctree, rd, destdir): + """Copy local files from srctree to given location. + Returns three-tuple of dicts: + 1. updated - files that already exist in SRCURI + 2. added - new files files that don't exist in SRCURI + 3 removed - files that exist in SRCURI but not in exported files + In each dict the key is the 'basepath' of the URI and value is the + absolute path to the existing file in recipe space (if any). + """ + import oe.recipeutils + + # Find out local files (SRC_URI files that exist in the "recipe space"). + # Local files that reside in srctree are not included in patch generation. + # Instead they are directly copied over the original source files (in + # recipe space). + existing_files = oe.recipeutils.get_recipe_local_files(rd) + new_set = None + updated = OrderedDict() + added = OrderedDict() + removed = OrderedDict() + local_files_dir = os.path.join(srctree, 'oe-local-files') + git_files = _git_ls_tree(srctree) + if 'oe-local-files' in git_files: + # If tracked by Git, take the files from srctree HEAD. First get + # the tree object of the directory + tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') + tree = git_files['oe-local-files'][2] + bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, + env=dict(os.environ, GIT_WORK_TREE=destdir, + GIT_INDEX_FILE=tmp_index)) + new_set = _git_ls_tree(srctree, tree, True).keys() + elif os.path.isdir(local_files_dir): + # If not tracked by Git, just copy from working copy + new_set = _ls_tree(os.path.join(srctree, 'oe-local-files')) + bb.process.run(['cp', '-ax', + os.path.join(srctree, 'oe-local-files', '.'), destdir]) + else: + new_set = [] + + # Special handling for kernel config + if bb.data.inherits_class('kernel-yocto', rd): + fragment_fn = 'devtool-fragment.cfg' + fragment_path = os.path.join(destdir, fragment_fn) + if _create_kconfig_diff(srctree, rd, fragment_path): + if os.path.exists(fragment_path): + if fragment_fn not in new_set: + new_set.append(fragment_fn) + # Copy fragment to local-files + if os.path.isdir(local_files_dir): + shutil.copy2(fragment_path, local_files_dir) + else: + if fragment_fn in new_set: + new_set.remove(fragment_fn) + # Remove fragment from local-files + if os.path.exists(os.path.join(local_files_dir, fragment_fn)): + os.unlink(os.path.join(local_files_dir, fragment_fn)) + + if new_set is not None: + for fname in new_set: + if fname in existing_files: + updated[fname] = existing_files.pop(fname) + elif fname != '.gitignore': + added[fname] = None + + removed = existing_files + return (updated, added, removed) + + +def _update_recipe_srcrev(args, srctree, rd, config_data): + """Implement the 'srcrev' mode of update-recipe""" + import bb + import oe.recipeutils + + recipefile = rd.getVar('FILE', True) + logger.info('Updating SRCREV in recipe %s' % os.path.basename(recipefile)) + + # Get HEAD revision + try: + stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) + except bb.process.ExecutionError as err: + raise DevtoolError('Failed to get HEAD revision in %s: %s' % + (srctree, err)) + srcrev = stdout.strip() + if len(srcrev) != 40: + raise DevtoolError('Invalid hash returned by git: %s' % stdout) + + destpath = None + remove_files = [] + patchfields = {} + patchfields['SRCREV'] = srcrev + orig_src_uri = rd.getVar('SRC_URI', False) or '' + srcuri = orig_src_uri.split() + tempdir = tempfile.mkdtemp(prefix='devtool') + update_srcuri = False + try: + local_files_dir = tempfile.mkdtemp(dir=tempdir) + upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir) + if not args.no_remove: + # Find list of existing patches in recipe file + patches_dir = tempfile.mkdtemp(dir=tempdir) + old_srcrev = (rd.getVar('SRCREV', False) or '') + upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, + patches_dir) + + # Remove deleted local files and "overlapping" patches + remove_files = del_f.values() + upd_p.values() + if remove_files: + removedentries = _remove_file_entries(srcuri, remove_files)[0] + update_srcuri = True + + if args.append: + files = dict((os.path.join(local_files_dir, key), val) for + key, val in upd_f.items() + new_f.items()) + removevalues = {} + if update_srcuri: + removevalues = {'SRC_URI': removedentries} + patchfields['SRC_URI'] = '\\\n '.join(srcuri) + _, destpath = oe.recipeutils.bbappend_recipe( + rd, args.append, files, wildcardver=args.wildcard_version, + extralines=patchfields, removevalues=removevalues) + else: + files_dir = os.path.join(os.path.dirname(recipefile), + rd.getVar('BPN', True)) + for basepath, path in upd_f.iteritems(): + logger.info('Updating file %s' % basepath) + _move_file(os.path.join(local_files_dir, basepath), path) + update_srcuri= True + for basepath, path in new_f.iteritems(): + logger.info('Adding new file %s' % basepath) + _move_file(os.path.join(local_files_dir, basepath), + os.path.join(files_dir, basepath)) + srcuri.append('file://%s' % basepath) + update_srcuri = True + if update_srcuri: + patchfields['SRC_URI'] = ' '.join(srcuri) + oe.recipeutils.patch_recipe(rd, recipefile, patchfields) + finally: + shutil.rmtree(tempdir) + if not 'git://' in orig_src_uri: + logger.info('You will need to update SRC_URI within the recipe to ' + 'point to a git repository where you have pushed your ' + 'changes') + + _remove_source_files(args, remove_files, destpath) + return True + +def _update_recipe_patch(args, config, workspace, srctree, rd, config_data): + """Implement the 'patch' mode of update-recipe""" + import bb + import oe.recipeutils + + recipefile = rd.getVar('FILE', True) + append = workspace[args.recipename]['bbappend'] + if not os.path.exists(append): + raise DevtoolError('unable to find workspace bbappend for recipe %s' % + args.recipename) + + initial_rev, update_rev, changed_revs = _get_patchset_revs(srctree, append, args.initial_rev) + if not initial_rev: + raise DevtoolError('Unable to find initial revision - please specify ' + 'it with --initial-rev') + + tempdir = tempfile.mkdtemp(prefix='devtool') + try: + local_files_dir = tempfile.mkdtemp(dir=tempdir) + upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir) + + remove_files = [] + if not args.no_remove: + # Get all patches from source tree and check if any should be removed + all_patches_dir = tempfile.mkdtemp(dir=tempdir) + upd_p, new_p, del_p = _export_patches(srctree, rd, initial_rev, + all_patches_dir) + # Remove deleted local files and patches + remove_files = del_f.values() + del_p.values() + + # Get updated patches from source tree + patches_dir = tempfile.mkdtemp(dir=tempdir) + upd_p, new_p, del_p = _export_patches(srctree, rd, update_rev, + patches_dir) + updatefiles = False + updaterecipe = False + destpath = None + srcuri = (rd.getVar('SRC_URI', False) or '').split() + if args.append: + files = dict((os.path.join(local_files_dir, key), val) for + key, val in upd_f.items() + new_f.items()) + files.update(dict((os.path.join(patches_dir, key), val) for + key, val in upd_p.items() + new_p.items())) + if files or remove_files: + removevalues = None + if remove_files: + removedentries, remaining = _remove_file_entries( + srcuri, remove_files) + if removedentries or remaining: + remaining = ['file://' + os.path.basename(item) for + item in remaining] + removevalues = {'SRC_URI': removedentries + remaining} + _, destpath = oe.recipeutils.bbappend_recipe( + rd, args.append, files, + wildcardver=args.wildcard_version, + removevalues=removevalues) + else: + logger.info('No patches or local source files needed updating') + else: + # Update existing files + for basepath, path in upd_f.iteritems(): + logger.info('Updating file %s' % basepath) + _move_file(os.path.join(local_files_dir, basepath), path) + updatefiles = True + for basepath, path in upd_p.iteritems(): + patchfn = os.path.join(patches_dir, basepath) + if changed_revs is not None: + # Avoid updating patches that have not actually changed + with open(patchfn, 'r') as f: + firstlineitems = f.readline().split() + if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: + if not firstlineitems[1] in changed_revs: + continue + logger.info('Updating patch %s' % basepath) + _move_file(patchfn, path) + updatefiles = True + # Add any new files + files_dir = os.path.join(os.path.dirname(recipefile), + rd.getVar('BPN', True)) + for basepath, path in new_f.iteritems(): + logger.info('Adding new file %s' % basepath) + _move_file(os.path.join(local_files_dir, basepath), + os.path.join(files_dir, basepath)) + srcuri.append('file://%s' % basepath) + updaterecipe = True + for basepath, path in new_p.iteritems(): + logger.info('Adding new patch %s' % basepath) + _move_file(os.path.join(patches_dir, basepath), + os.path.join(files_dir, basepath)) + srcuri.append('file://%s' % basepath) + updaterecipe = True + # Update recipe, if needed + if _remove_file_entries(srcuri, remove_files)[0]: + updaterecipe = True + if updaterecipe: + logger.info('Updating recipe %s' % os.path.basename(recipefile)) + oe.recipeutils.patch_recipe(rd, recipefile, + {'SRC_URI': ' '.join(srcuri)}) + elif not updatefiles: + # Neither patches nor recipe were updated + logger.info('No patches or files need updating') + return False + finally: + shutil.rmtree(tempdir) + + _remove_source_files(args, remove_files, destpath) + return True + +def _guess_recipe_update_mode(srctree, rdata): + """Guess the recipe update mode to use""" + src_uri = (rdata.getVar('SRC_URI', False) or '').split() + git_uris = [uri for uri in src_uri if uri.startswith('git://')] + if not git_uris: + return 'patch' + # Just use the first URI for now + uri = git_uris[0] + # Check remote branch + params = bb.fetch.decodeurl(uri)[5] + upstr_branch = params['branch'] if 'branch' in params else 'master' + # Check if current branch HEAD is found in upstream branch + stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) + head_rev = stdout.rstrip() + stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, + cwd=srctree) + remote_brs = [branch.strip() for branch in stdout.splitlines()] + if 'origin/' + upstr_branch in remote_brs: + return 'srcrev' + + return 'patch' + +def update_recipe(args, config, basepath, workspace): + """Entry point for the devtool 'update-recipe' subcommand""" + check_workspace_recipe(workspace, args.recipename) + + if args.append: + if not os.path.exists(args.append): + raise DevtoolError('bbappend destination layer directory "%s" ' + 'does not exist' % args.append) + if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): + raise DevtoolError('conf/layer.conf not found in bbappend ' + 'destination layer "%s"' % args.append) + + tinfoil = setup_tinfoil(basepath=basepath, tracking=True) + + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + + srctree = workspace[args.recipename]['srctree'] + if args.mode == 'auto': + mode = _guess_recipe_update_mode(srctree, rd) + else: + mode = args.mode + + if mode == 'srcrev': + updated = _update_recipe_srcrev(args, srctree, rd, tinfoil.config_data) + elif mode == 'patch': + updated = _update_recipe_patch(args, config, workspace, srctree, rd, tinfoil.config_data) + else: + raise DevtoolError('update_recipe: invalid mode %s' % mode) + + if updated: + rf = rd.getVar('FILE', True) + if rf.startswith(config.workspace_path): + logger.warn('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) + + return 0 + + +def status(args, config, basepath, workspace): + """Entry point for the devtool 'status' subcommand""" + if workspace: + for recipe, value in workspace.iteritems(): + recipefile = value['recipefile'] + if recipefile: + recipestr = ' (%s)' % recipefile + else: + recipestr = '' + print("%s: %s%s" % (recipe, value['srctree'], recipestr)) + else: + logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') + return 0 + + +def reset(args, config, basepath, workspace): + """Entry point for the devtool 'reset' subcommand""" + import bb + if args.recipename: + if args.all: + raise DevtoolError("Recipe cannot be specified if -a/--all is used") + else: + check_workspace_recipe(workspace, args.recipename, checksrc=False) + elif not args.all: + raise DevtoolError("Recipe must be specified, or specify -a/--all to " + "reset all recipes") + if args.all: + recipes = workspace.keys() + else: + recipes = [args.recipename] + + if recipes and not args.no_clean: + if len(recipes) == 1: + logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) + else: + logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) + # If the recipe file itself was created in the workspace, and + # it uses BBCLASSEXTEND, then we need to also clean the other + # variants + targets = [] + for recipe in recipes: + targets.append(recipe) + recipefile = workspace[recipe]['recipefile'] + if recipefile and os.path.exists(recipefile): + targets.extend(get_bbclassextend_targets(recipefile, recipe)) + try: + exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) + except bb.process.ExecutionError as e: + raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' + 'wish, you may specify -n/--no-clean to ' + 'skip running this command when resetting' % + (e.command, e.stdout)) + + for pn in recipes: + _check_preserve(config, pn) + + preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) + def preservedir(origdir): + if os.path.exists(origdir): + for root, dirs, files in os.walk(origdir): + for fn in files: + logger.warn('Preserving %s in %s' % (fn, preservepath)) + _move_file(os.path.join(origdir, fn), + os.path.join(preservepath, fn)) + for dn in dirs: + preservedir(os.path.join(root, dn)) + os.rmdir(origdir) + + preservedir(os.path.join(config.workspace_path, 'recipes', pn)) + # We don't automatically create this dir next to appends, but the user can + preservedir(os.path.join(config.workspace_path, 'appends', pn)) + + srctree = workspace[pn]['srctree'] + if os.path.isdir(srctree): + if os.listdir(srctree): + # We don't want to risk wiping out any work in progress + logger.info('Leaving source tree %s as-is; if you no ' + 'longer need it then please delete it manually' + % srctree) + else: + # This is unlikely, but if it's empty we can just remove it + os.rmdir(srctree) + + return 0 + + +def get_default_srctree(config, recipename=''): + """Get the default srctree path""" + srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) + if recipename: + return os.path.join(srctreeparent, 'sources', recipename) + else: + return os.path.join(srctreeparent, 'sources') + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + + defsrctree = get_default_srctree(context.config) + parser_add = subparsers.add_parser('add', help='Add a new recipe', + description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', + group='starting', order=100) + parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') + parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) + parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') + group = parser_add.add_mutually_exclusive_group() + group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") + group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") + parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') + parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') + parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") + parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') + parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') + parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') + parser_add.set_defaults(func=add) + + parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', + description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', + group='starting', order=90) + parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') + parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) + parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') + group = parser_modify.add_mutually_exclusive_group() + group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') + group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') + group = parser_modify.add_mutually_exclusive_group() + group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") + group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") + parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') + parser_modify.set_defaults(func=modify) + + parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', + description='Extracts the source for an existing recipe', + group='advanced') + parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') + parser_extract.add_argument('srctree', help='Path to where to extract the source tree') + parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') + parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') + parser_extract.set_defaults(func=extract, no_workspace=True) + + parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', + description='Synchronize the previously extracted source tree for an existing recipe', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + group='advanced') + parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') + parser_sync.add_argument('srctree', help='Path to the source tree') + parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') + parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') + parser_sync.set_defaults(func=sync) + + parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', + description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', + group='working', order=-90) + parser_update_recipe.add_argument('recipename', help='Name of recipe to update') + parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') + parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') + parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') + parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') + parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') + parser_update_recipe.set_defaults(func=update_recipe) + + parser_status = subparsers.add_parser('status', help='Show workspace status', + description='Lists recipes currently in your workspace and the paths to their respective external source trees', + group='info', order=100) + parser_status.set_defaults(func=status) + + parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', + description='Removes the specified recipe from your workspace (resetting its state)', + group='working', order=-100) + parser_reset.add_argument('recipename', nargs='?', help='Recipe to reset') + parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') + parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') + parser_reset.set_defaults(func=reset) diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py new file mode 100644 index 0000000..a085f78 --- /dev/null +++ b/scripts/lib/devtool/upgrade.py @@ -0,0 +1,382 @@ +# Development tool - upgrade command plugin +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +"""Devtool upgrade plugin""" + +import os +import sys +import re +import shutil +import tempfile +import logging +import argparse +import scriptutils +import errno +import bb +import oe.recipeutils +from devtool import standard +from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build + +logger = logging.getLogger('devtool') + +def _run(cmd, cwd=''): + logger.debug("Running command %s> %s" % (cwd,cmd)) + return bb.process.run('%s' % cmd, cwd=cwd) + +def _get_srctree(tmpdir): + srctree = tmpdir + dirs = os.listdir(tmpdir) + if len(dirs) == 1: + srctree = os.path.join(tmpdir, dirs[0]) + return srctree + +def _copy_source_code(orig, dest): + for path in standard._ls_tree(orig): + dest_dir = os.path.join(dest, os.path.dirname(path)) + bb.utils.mkdirhier(dest_dir) + dest_path = os.path.join(dest, path) + shutil.move(os.path.join(orig, path), dest_path) + +def _get_checksums(rf): + import re + checksums = {} + with open(rf) as f: + for line in f: + for cs in ['md5sum', 'sha256sum']: + m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line) + if m: + checksums[cs] = m.group(1) + return checksums + +def _remove_patch_dirs(recipefolder): + for root, dirs, files in os.walk(recipefolder): + for d in dirs: + shutil.rmtree(os.path.join(root,d)) + +def _recipe_contains(rd, var): + rf = rd.getVar('FILE', True) + varfiles = oe.recipeutils.get_var_files(rf, [var], rd) + for var, fn in varfiles.iteritems(): + if fn and fn.startswith(os.path.dirname(rf) + os.sep): + return True + return False + +def _rename_recipe_dirs(oldpv, newpv, path): + for root, dirs, files in os.walk(path): + for olddir in dirs: + if olddir.find(oldpv) != -1: + newdir = olddir.replace(oldpv, newpv) + if olddir != newdir: + shutil.move(os.path.join(path, olddir), os.path.join(path, newdir)) + +def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path): + oldrecipe = os.path.basename(oldrecipe) + if oldrecipe.endswith('_%s.bb' % oldpv): + newrecipe = '%s_%s.bb' % (bpn, newpv) + if oldrecipe != newrecipe: + shutil.move(os.path.join(path, oldrecipe), os.path.join(path, newrecipe)) + else: + newrecipe = oldrecipe + return os.path.join(path, newrecipe) + +def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path): + _rename_recipe_dirs(oldpv, newpv, path) + return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path) + +def _write_append(rc, srctree, same_dir, no_same_dir, rev, workspace, d): + """Writes an append file""" + if not os.path.exists(rc): + raise DevtoolError("bbappend not created because %s does not exist" % rc) + + appendpath = os.path.join(workspace, 'appends') + if not os.path.exists(appendpath): + bb.utils.mkdirhier(appendpath) + + brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename + + srctree = os.path.abspath(srctree) + pn = d.getVar('PN',True) + af = os.path.join(appendpath, '%s.bbappend' % brf) + with open(af, 'w') as f: + f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n') + f.write('inherit externalsrc\n') + f.write(('# NOTE: We use pn- overrides here to avoid affecting' + 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n')) + f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) + b_is_s = use_external_build(same_dir, no_same_dir, d) + if b_is_s: + f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) + if rev: + f.write('\n# initial_rev: %s\n' % rev) + return af + +def _cleanup_on_error(rf, srctree): + rfp = os.path.split(rf)[0] # recipe folder + rfpp = os.path.split(rfp)[0] # recipes folder + if os.path.exists(rfp): + shutil.rmtree(b) + if not len(os.listdir(rfpp)): + os.rmdir(rfpp) + srctree = os.path.abspath(srctree) + if os.path.exists(srctree): + shutil.rmtree(srctree) + +def _upgrade_error(e, rf, srctree): + if rf: + cleanup_on_error(rf, srctree) + logger.error(e) + raise DevtoolError(e) + +def _get_uri(rd): + srcuris = rd.getVar('SRC_URI', True).split() + if not len(srcuris): + raise DevtoolError('SRC_URI not found on recipe') + # Get first non-local entry in SRC_URI - usually by convention it's + # the first entry, but not always! + srcuri = None + for entry in srcuris: + if not entry.startswith('file://'): + srcuri = entry + break + if not srcuri: + raise DevtoolError('Unable to find non-local entry in SRC_URI') + srcrev = '${AUTOREV}' + if '://' in srcuri: + # Fetch a URL + rev_re = re.compile(';rev=([^;]+)') + res = rev_re.search(srcuri) + if res: + srcrev = res.group(1) + srcuri = rev_re.sub('', srcuri) + return srcuri, srcrev + +def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd): + """Extract sources of a recipe with a new version""" + + def __run(cmd): + """Simple wrapper which calls _run with srctree as cwd""" + return _run(cmd, srctree) + + crd = rd.createCopy() + + pv = crd.getVar('PV', True) + crd.setVar('PV', newpv) + + tmpsrctree = None + uri, rev = _get_uri(crd) + if srcrev: + rev = srcrev + if uri.startswith('git://'): + __run('git fetch') + __run('git checkout %s' % rev) + __run('git tag -f devtool-base-new') + md5 = None + sha256 = None + else: + __run('git checkout devtool-base -b devtool-%s' % newpv) + + tmpdir = tempfile.mkdtemp(prefix='devtool') + try: + md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev) + except bb.fetch2.FetchError as e: + raise DevtoolError(e) + + tmpsrctree = _get_srctree(tmpdir) + srctree = os.path.abspath(srctree) + + # Delete all sources so we ensure no stray files are left over + for item in os.listdir(srctree): + if item in ['.git', 'oe-local-files']: + continue + itempath = os.path.join(srctree, item) + if os.path.isdir(itempath): + shutil.rmtree(itempath) + else: + os.remove(itempath) + + # Copy in new ones + _copy_source_code(tmpsrctree, srctree) + + (stdout,_) = __run('git ls-files --modified --others --exclude-standard') + for f in stdout.splitlines(): + __run('git add "%s"' % f) + + __run('git commit -q -m "Commit of upstream changes at version %s" --allow-empty' % newpv) + __run('git tag -f devtool-base-%s' % newpv) + + (stdout, _) = __run('git rev-parse HEAD') + rev = stdout.rstrip() + + if no_patch: + patches = oe.recipeutils.get_recipe_patches(crd) + if len(patches): + logger.warn('By user choice, the following patches will NOT be applied') + for patch in patches: + logger.warn("%s" % os.path.basename(patch)) + else: + try: + __run('git checkout devtool-patched -b %s' % branch) + __run('git rebase %s' % rev) + if uri.startswith('git://'): + suffix = 'new' + else: + suffix = newpv + __run('git tag -f devtool-patched-%s' % suffix) + except bb.process.ExecutionError as e: + logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) + + if tmpsrctree: + if keep_temp: + logger.info('Preserving temporary directory %s' % tmpsrctree) + else: + shutil.rmtree(tmpsrctree) + + return (rev, md5, sha256) + +def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil, rd): + """Creates the new recipe under workspace""" + + bpn = rd.getVar('BPN', True) + path = os.path.join(workspace, 'recipes', bpn) + bb.utils.mkdirhier(path) + oe.recipeutils.copy_recipe_files(rd, path) + + oldpv = rd.getVar('PV', True) + if not newpv: + newpv = oldpv + origpath = rd.getVar('FILE', True) + fullpath = _rename_recipe_files(origpath, bpn, oldpv, newpv, path) + logger.debug('Upgraded %s => %s' % (origpath, fullpath)) + + newvalues = {} + if _recipe_contains(rd, 'PV') and newpv != oldpv: + newvalues['PV'] = newpv + + if srcrev: + newvalues['SRCREV'] = srcrev + + if srcbranch: + src_uri = oe.recipeutils.split_var_value(rd.getVar('SRC_URI', False) or '') + changed = False + replacing = True + new_src_uri = [] + for entry in src_uri: + scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry) + if replacing and scheme in ['git', 'gitsm']: + branch = params.get('branch', 'master') + if rd.expand(branch) != srcbranch: + # Handle case where branch is set through a variable + res = re.match(r'\$\{([^}@]+)\}', branch) + if res: + newvalues[res.group(1)] = srcbranch + # We know we won't change SRC_URI now, so break out + break + else: + params['branch'] = srcbranch + entry = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params)) + changed = True + replacing = False + new_src_uri.append(entry) + if changed: + newvalues['SRC_URI'] = ' '.join(new_src_uri) + + newvalues['PR'] = None + + if md5 and sha256: + newvalues['SRC_URI[md5sum]'] = md5 + newvalues['SRC_URI[sha256sum]'] = sha256 + + rd = oe.recipeutils.parse_recipe(fullpath, None, tinfoil.config_data) + oe.recipeutils.patch_recipe(rd, fullpath, newvalues) + + return fullpath + +def upgrade(args, config, basepath, workspace): + """Entry point for the devtool 'upgrade' subcommand""" + + if args.recipename in workspace: + raise DevtoolError("recipe %s is already in your workspace" % args.recipename) + if not args.version and not args.srcrev: + raise DevtoolError("You must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option") + if args.srcbranch and not args.srcrev: + raise DevtoolError("If you specify --srcbranch/-B then you must use --srcrev/-S to specify the revision" % args.recipename) + + tinfoil = setup_tinfoil(basepath=basepath, tracking=True) + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + + pn = rd.getVar('PN', True) + if pn != args.recipename: + logger.info('Mapping %s to %s' % (args.recipename, pn)) + if pn in workspace: + raise DevtoolError("recipe %s is already in your workspace" % pn) + + if args.srctree: + srctree = os.path.abspath(args.srctree) + else: + srctree = standard.get_default_srctree(config, pn) + + standard._check_compatible_recipe(pn, rd) + old_srcrev = rd.getVar('SRCREV', True) + if old_srcrev == 'INVALID': + old_srcrev = None + if old_srcrev and not args.srcrev: + raise DevtoolError("Recipe specifies a SRCREV value; you must specify a new one when upgrading") + if rd.getVar('PV', True) == args.version and old_srcrev == args.srcrev: + raise DevtoolError("Current and upgrade versions are the same version") + + rf = None + try: + rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd) + rev2, md5, sha256 = _extract_new_source(args.version, srctree, args.no_patch, + args.srcrev, args.branch, args.keep_temp, + tinfoil, rd) + rf = _create_new_recipe(args.version, md5, sha256, args.srcrev, args.srcbranch, config.workspace_path, tinfoil, rd) + except bb.process.CmdError as e: + _upgrade_error(e, rf, srctree) + except DevtoolError as e: + _upgrade_error(e, rf, srctree) + standard._add_md5(config, pn, os.path.dirname(rf)) + + af = _write_append(rf, srctree, args.same_dir, args.no_same_dir, rev2, + config.workspace_path, rd) + standard._add_md5(config, pn, af) + logger.info('Upgraded source extracted to %s' % srctree) + logger.info('New recipe is %s' % rf) + return 0 + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + + defsrctree = standard.get_default_srctree(context.config) + + parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe', + description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).', + group='starting') + parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)') + parser_upgrade.add_argument('srctree', nargs='?', help='Path to where to extract the source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) + parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)') + parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (if fetching from an SCM such as git)') + parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)') + parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")') + parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code') + group = parser_upgrade.add_mutually_exclusive_group() + group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") + group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") + parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') + parser_upgrade.set_defaults(func=upgrade) diff --git a/scripts/lib/devtool/utilcmds.py b/scripts/lib/devtool/utilcmds.py new file mode 100644 index 0000000..b761a80 --- /dev/null +++ b/scripts/lib/devtool/utilcmds.py @@ -0,0 +1,233 @@ +# Development tool - utility commands plugin +# +# Copyright (C) 2015-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +"""Devtool utility plugins""" + +import os +import sys +import shutil +import tempfile +import logging +import argparse +import subprocess +import scriptutils +from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, DevtoolError +from devtool import parse_recipe + +logger = logging.getLogger('devtool') + + +def edit_recipe(args, config, basepath, workspace): + """Entry point for the devtool 'edit-recipe' subcommand""" + if args.any_recipe: + tinfoil = setup_tinfoil(config_only=False, basepath=basepath) + try: + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + recipefile = rd.getVar('FILE', True) + finally: + tinfoil.shutdown() + else: + check_workspace_recipe(workspace, args.recipename) + recipefile = workspace[args.recipename]['recipefile'] + if not recipefile: + raise DevtoolError("Recipe file for %s is not under the workspace" % + args.recipename) + + return scriptutils.run_editor(recipefile) + + +def configure_help(args, config, basepath, workspace): + """Entry point for the devtool 'configure-help' subcommand""" + import oe.utils + + check_workspace_recipe(workspace, args.recipename) + tinfoil = setup_tinfoil(config_only=False, basepath=basepath) + try: + rd = parse_recipe(config, tinfoil, args.recipename, appends=True, filter_workspace=False) + if not rd: + return 1 + b = rd.getVar('B', True) + s = rd.getVar('S', True) + configurescript = os.path.join(s, 'configure') + confdisabled = 'noexec' in rd.getVarFlags('do_configure') or 'do_configure' not in (rd.getVar('__BBTASKS', False) or []) + configureopts = oe.utils.squashspaces(rd.getVar('CONFIGUREOPTS', True) or '') + extra_oeconf = oe.utils.squashspaces(rd.getVar('EXTRA_OECONF', True) or '') + extra_oecmake = oe.utils.squashspaces(rd.getVar('EXTRA_OECMAKE', True) or '') + do_configure = rd.getVar('do_configure', True) or '' + do_configure_noexpand = rd.getVar('do_configure', False) or '' + packageconfig = rd.getVarFlags('PACKAGECONFIG') or [] + autotools = bb.data.inherits_class('autotools', rd) and ('oe_runconf' in do_configure or 'autotools_do_configure' in do_configure) + cmake = bb.data.inherits_class('cmake', rd) and ('cmake_do_configure' in do_configure) + cmake_do_configure = rd.getVar('cmake_do_configure', True) + pn = rd.getVar('PN', True) + finally: + tinfoil.shutdown() + + if 'doc' in packageconfig: + del packageconfig['doc'] + + if autotools and not os.path.exists(configurescript): + logger.info('Running do_configure to generate configure script') + try: + stdout, _ = exec_build_env_command(config.init_path, basepath, + 'bitbake -c configure %s' % args.recipename, + stderr=subprocess.STDOUT) + except bb.process.ExecutionError: + pass + + if confdisabled or do_configure.strip() in ('', ':'): + raise DevtoolError("do_configure task has been disabled for this recipe") + elif args.no_pager and not os.path.exists(configurescript): + raise DevtoolError("No configure script found and no other information to display") + else: + configopttext = '' + if autotools and configureopts: + configopttext = ''' +Arguments currently passed to the configure script: + +%s + +Some of those are fixed.''' % (configureopts + ' ' + extra_oeconf) + if extra_oeconf: + configopttext += ''' The ones that are specified through EXTRA_OECONF (which you can change or add to easily): + +%s''' % extra_oeconf + + elif cmake: + in_cmake = False + cmake_cmd = '' + for line in cmake_do_configure.splitlines(): + if in_cmake: + cmake_cmd = cmake_cmd + ' ' + line.strip().rstrip('\\') + if not line.endswith('\\'): + break + if line.lstrip().startswith('cmake '): + cmake_cmd = line.strip().rstrip('\\') + if line.endswith('\\'): + in_cmake = True + else: + break + if cmake_cmd: + configopttext = ''' +The current cmake command line: + +%s + +Arguments specified through EXTRA_OECMAKE (which you can change or add to easily) + +%s''' % (oe.utils.squashspaces(cmake_cmd), extra_oecmake) + else: + configopttext = ''' +The current implementation of cmake_do_configure: + +cmake_do_configure() { +%s +} + +Arguments specified through EXTRA_OECMAKE (which you can change or add to easily) + +%s''' % (cmake_do_configure.rstrip(), extra_oecmake) + + elif do_configure: + configopttext = ''' +The current implementation of do_configure: + +do_configure() { +%s +}''' % do_configure.rstrip() + if '${EXTRA_OECONF}' in do_configure_noexpand: + configopttext += ''' + +Arguments specified through EXTRA_OECONF (which you can change or add to easily): + +%s''' % extra_oeconf + + if packageconfig: + configopttext += ''' + +Some of these options may be controlled through PACKAGECONFIG; for more details please see the recipe.''' + + if args.arg: + helpargs = ' '.join(args.arg) + elif cmake: + helpargs = '-LH' + else: + helpargs = '--help' + + msg = '''configure information for %s +------------------------------------------ +%s''' % (pn, configopttext) + + if cmake: + msg += ''' + +The cmake %s output for %s follows. After "-- Cache values" you should see a list of variables you can add to EXTRA_OECMAKE (prefixed with -D and suffixed with = followed by the desired value, without any spaces). +------------------------------------------''' % (helpargs, pn) + elif os.path.exists(configurescript): + msg += ''' + +The ./configure %s output for %s follows. +------------------------------------------''' % (helpargs, pn) + + olddir = os.getcwd() + tmppath = tempfile.mkdtemp() + with tempfile.NamedTemporaryFile('w', delete=False) as tf: + if not args.no_header: + tf.write(msg + '\n') + tf.close() + try: + try: + cmd = 'cat %s' % tf.name + if cmake: + cmd += '; cmake %s %s 2>&1' % (helpargs, s) + os.chdir(b) + elif os.path.exists(configurescript): + cmd += '; %s %s' % (configurescript, helpargs) + if sys.stdout.isatty() and not args.no_pager: + pager = os.environ.get('PAGER', 'less') + cmd = '(%s) | %s' % (cmd, pager) + subprocess.check_call(cmd, shell=True) + except subprocess.CalledProcessError as e: + return e.returncode + finally: + os.chdir(olddir) + shutil.rmtree(tmppath) + os.remove(tf.name) + + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + parser_edit_recipe = subparsers.add_parser('edit-recipe', help='Edit a recipe file in your workspace', + description='Runs the default editor (as specified by the EDITOR variable) on the specified recipe. Note that the recipe file itself must be in the workspace (i.e. as a result of "devtool add" or "devtool upgrade"); you can override this with the -a/--any-recipe option.', + group='working') + parser_edit_recipe.add_argument('recipename', help='Recipe to edit') + parser_edit_recipe.add_argument('--any-recipe', '-a', action="store_true", help='Edit any recipe, not just where the recipe file itself is in the workspace') + parser_edit_recipe.set_defaults(func=edit_recipe) + + # NOTE: Needed to override the usage string here since the default + # gets the order wrong - recipename must come before --arg + parser_configure_help = subparsers.add_parser('configure-help', help='Get help on configure script options', + usage='devtool configure-help [options] recipename [--arg ...]', + description='Displays the help for the configure script for the specified recipe (i.e. runs ./configure --help) prefaced by a header describing the current options being specified. Output is piped through less (or whatever PAGER is set to, if set) for easy browsing.', + group='working') + parser_configure_help.add_argument('recipename', help='Recipe to show configure help for') + parser_configure_help.add_argument('-p', '--no-pager', help='Disable paged output', action="store_true") + parser_configure_help.add_argument('-n', '--no-header', help='Disable explanatory header text', action="store_true") + parser_configure_help.add_argument('--arg', help='Pass remaining arguments to the configure script instead of --help (useful if the script has additional help options)', nargs=argparse.REMAINDER) + parser_configure_help.set_defaults(func=configure_help) diff --git a/scripts/lib/recipetool/__init__.py b/scripts/lib/recipetool/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/scripts/lib/recipetool/__init__.py diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py new file mode 100644 index 0000000..558fd25 --- /dev/null +++ b/scripts/lib/recipetool/append.py @@ -0,0 +1,471 @@ +# Recipe creation tool - append plugin +# +# Copyright (C) 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import os +import argparse +import glob +import fnmatch +import re +import subprocess +import logging +import stat +import shutil +import scriptutils +import errno +from collections import defaultdict + +logger = logging.getLogger('recipetool') + +tinfoil = None + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +# FIXME guessing when we don't have pkgdata? +# FIXME mode to create patch rather than directly substitute + +class InvalidTargetFileError(Exception): + pass + +def find_target_file(targetpath, d, pkglist=None): + """Find the recipe installing the specified target path, optionally limited to a select list of packages""" + import json + + pkgdata_dir = d.getVar('PKGDATA_DIR', True) + + # The mix between /etc and ${sysconfdir} here may look odd, but it is just + # being consistent with usage elsewhere + invalidtargets = {'${sysconfdir}/version': '${sysconfdir}/version is written out at image creation time', + '/etc/timestamp': '/etc/timestamp is written out at image creation time', + '/dev/*': '/dev is handled by udev (or equivalent) and the kernel (devtmpfs)', + '/etc/passwd': '/etc/passwd should be managed through the useradd and extrausers classes', + '/etc/group': '/etc/group should be managed through the useradd and extrausers classes', + '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes', + '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes', + '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',} + + for pthspec, message in invalidtargets.iteritems(): + if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)): + raise InvalidTargetFileError(d.expand(message)) + + targetpath_re = re.compile(r'\s+(\$D)?%s(\s|$)' % targetpath) + + recipes = defaultdict(list) + for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')): + if pkglist: + filelist = pkglist + else: + filelist = files + for fn in filelist: + pkgdatafile = os.path.join(root, fn) + if pkglist and not os.path.exists(pkgdatafile): + continue + with open(pkgdatafile, 'r') as f: + pn = '' + # This does assume that PN comes before other values, but that's a fairly safe assumption + for line in f: + if line.startswith('PN:'): + pn = line.split(':', 1)[1].strip() + elif line.startswith('FILES_INFO:'): + val = line.split(':', 1)[1].strip() + dictval = json.loads(val) + for fullpth in dictval.keys(): + if fnmatch.fnmatchcase(fullpth, targetpath): + recipes[targetpath].append(pn) + elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'): + scriptval = line.split(':', 1)[1].strip().decode('string_escape') + if 'update-alternatives --install %s ' % targetpath in scriptval: + recipes[targetpath].append('?%s' % pn) + elif targetpath_re.search(scriptval): + recipes[targetpath].append('!%s' % pn) + return recipes + +def _get_recipe_file(cooker, pn): + import oe.recipeutils + recipefile = oe.recipeutils.pn_to_recipe(cooker, pn) + if not recipefile: + skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn) + if skipreasons: + logger.error('\n'.join(skipreasons)) + else: + logger.error("Unable to find any recipe file matching %s" % pn) + return recipefile + +def _parse_recipe(pn, tinfoil): + import oe.recipeutils + recipefile = _get_recipe_file(tinfoil.cooker, pn) + if not recipefile: + # Error already logged + return None + append_files = tinfoil.cooker.collection.get_file_appends(recipefile) + rd = oe.recipeutils.parse_recipe(recipefile, append_files, + tinfoil.config_data) + return rd + +def determine_file_source(targetpath, rd): + """Assuming we know a file came from a specific recipe, figure out exactly where it came from""" + import oe.recipeutils + + # See if it's in do_install for the recipe + workdir = rd.getVar('WORKDIR', True) + src_uri = rd.getVar('SRC_URI', True) + srcfile = '' + modpatches = [] + elements = check_do_install(rd, targetpath) + if elements: + logger.debug('do_install line:\n%s' % ' '.join(elements)) + srcpath = get_source_path(elements) + logger.debug('source path: %s' % srcpath) + if not srcpath.startswith('/'): + # Handle non-absolute path + srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs', True).split()[-1], srcpath)) + if srcpath.startswith(workdir): + # OK, now we have the source file name, look for it in SRC_URI + workdirfile = os.path.relpath(srcpath, workdir) + # FIXME this is where we ought to have some code in the fetcher, because this is naive + for item in src_uri.split(): + localpath = bb.fetch2.localpath(item, rd) + # Source path specified in do_install might be a glob + if fnmatch.fnmatch(os.path.basename(localpath), workdirfile): + srcfile = 'file://%s' % localpath + elif '/' in workdirfile: + if item == 'file://%s' % workdirfile: + srcfile = 'file://%s' % localpath + + # Check patches + srcpatches = [] + patchedfiles = oe.recipeutils.get_recipe_patched_files(rd) + for patch, filelist in patchedfiles.iteritems(): + for fileitem in filelist: + if fileitem[0] == srcpath: + srcpatches.append((patch, fileitem[1])) + if srcpatches: + addpatch = None + for patch in srcpatches: + if patch[1] == 'A': + addpatch = patch[0] + else: + modpatches.append(patch[0]) + if addpatch: + srcfile = 'patch://%s' % addpatch + + return (srcfile, elements, modpatches) + +def get_source_path(cmdelements): + """Find the source path specified within a command""" + command = cmdelements[0] + if command in ['install', 'cp']: + helptext = subprocess.check_output('LC_ALL=C %s --help' % command, shell=True) + argopts = '' + argopt_line_re = re.compile('^-([a-zA-Z0-9]), --[a-z-]+=') + for line in helptext.splitlines(): + line = line.lstrip() + res = argopt_line_re.search(line) + if res: + argopts += res.group(1) + if not argopts: + # Fallback + if command == 'install': + argopts = 'gmoSt' + elif command == 'cp': + argopts = 't' + else: + raise Exception('No fallback arguments for command %s' % command) + + skipnext = False + for elem in cmdelements[1:-1]: + if elem.startswith('-'): + if len(elem) > 1 and elem[1] in argopts: + skipnext = True + continue + if skipnext: + skipnext = False + continue + return elem + else: + raise Exception('get_source_path: no handling for command "%s"') + +def get_func_deps(func, d): + """Find the function dependencies of a shell function""" + deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True)) + deps |= set((d.getVarFlag(func, "vardeps", True) or "").split()) + funcdeps = [] + for dep in deps: + if d.getVarFlag(dep, 'func', True): + funcdeps.append(dep) + return funcdeps + +def check_do_install(rd, targetpath): + """Look at do_install for a command that installs/copies the specified target path""" + instpath = os.path.abspath(os.path.join(rd.getVar('D', True), targetpath.lstrip('/'))) + do_install = rd.getVar('do_install', True) + # Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose) + deps = get_func_deps('do_install', rd) + for dep in deps: + do_install = do_install.replace(dep, rd.getVar(dep, True)) + + # Look backwards through do_install as we want to catch where a later line (perhaps + # from a bbappend) is writing over the top + for line in reversed(do_install.splitlines()): + line = line.strip() + if (line.startswith('install ') and ' -m' in line) or line.startswith('cp '): + elements = line.split() + destpath = os.path.abspath(elements[-1]) + if destpath == instpath: + return elements + elif destpath.rstrip('/') == os.path.dirname(instpath): + # FIXME this doesn't take recursive copy into account; unsure if it's practical to do so + srcpath = get_source_path(elements) + if fnmatch.fnmatchcase(os.path.basename(instpath), os.path.basename(srcpath)): + return elements + return None + + +def appendfile(args): + import oe.recipeutils + + stdout = '' + try: + (stdout, _) = bb.process.run('LANG=C file -b %s' % args.newfile, shell=True) + if 'cannot open' in stdout: + raise bb.process.ExecutionError(stdout) + except bb.process.ExecutionError as err: + logger.debug('file command returned error: %s' % err) + stdout = '' + if stdout: + logger.debug('file command output: %s' % stdout.rstrip()) + if ('executable' in stdout and not 'shell script' in stdout) or 'shared object' in stdout: + logger.warn('This file looks like it is a binary or otherwise the output of compilation. If it is, you should consider building it properly instead of substituting a binary file directly.') + + if args.recipe: + recipes = {args.targetpath: [args.recipe],} + else: + try: + recipes = find_target_file(args.targetpath, tinfoil.config_data) + except InvalidTargetFileError as e: + logger.error('%s cannot be handled by this tool: %s' % (args.targetpath, e)) + return 1 + if not recipes: + logger.error('Unable to find any package producing path %s - this may be because the recipe packaging it has not been built yet' % args.targetpath) + return 1 + + alternative_pns = [] + postinst_pns = [] + + selectpn = None + for targetpath, pnlist in recipes.iteritems(): + for pn in pnlist: + if pn.startswith('?'): + alternative_pns.append(pn[1:]) + elif pn.startswith('!'): + postinst_pns.append(pn[1:]) + elif selectpn: + # hit here with multilibs + continue + else: + selectpn = pn + + if not selectpn and len(alternative_pns) == 1: + selectpn = alternative_pns[0] + logger.error('File %s is an alternative possibly provided by recipe %s but seemingly no other, selecting it by default - you should double check other recipes' % (args.targetpath, selectpn)) + + if selectpn: + logger.debug('Selecting recipe %s for file %s' % (selectpn, args.targetpath)) + if postinst_pns: + logger.warn('%s be modified by postinstall scripts for the following recipes:\n %s\nThis may or may not be an issue depending on what modifications these postinstall scripts make.' % (args.targetpath, '\n '.join(postinst_pns))) + rd = _parse_recipe(selectpn, tinfoil) + if not rd: + # Error message already shown + return 1 + sourcefile, instelements, modpatches = determine_file_source(args.targetpath, rd) + sourcepath = None + if sourcefile: + sourcetype, sourcepath = sourcefile.split('://', 1) + logger.debug('Original source file is %s (%s)' % (sourcepath, sourcetype)) + if sourcetype == 'patch': + logger.warn('File %s is added by the patch %s - you may need to remove or replace this patch in order to replace the file.' % (args.targetpath, sourcepath)) + sourcepath = None + else: + logger.debug('Unable to determine source file, proceeding anyway') + if modpatches: + logger.warn('File %s is modified by the following patches:\n %s' % (args.targetpath, '\n '.join(modpatches))) + + if instelements and sourcepath: + install = None + else: + # Auto-determine permissions + # Check destination + binpaths = '${bindir}:${sbindir}:${base_bindir}:${base_sbindir}:${libexecdir}:${sysconfdir}/init.d' + perms = '0644' + if os.path.abspath(os.path.dirname(args.targetpath)) in rd.expand(binpaths).split(':'): + # File is going into a directory normally reserved for executables, so it should be executable + perms = '0755' + else: + # Check source + st = os.stat(args.newfile) + if st.st_mode & stat.S_IXUSR: + perms = '0755' + install = {args.newfile: (args.targetpath, perms)} + oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine) + return 0 + else: + if alternative_pns: + logger.error('File %s is an alternative possibly provided by the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(alternative_pns))) + elif postinst_pns: + logger.error('File %s may be written out in a pre/postinstall script of the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(postinst_pns))) + return 3 + + +def appendsrc(args, files, rd, extralines=None): + import oe.recipeutils + + srcdir = rd.getVar('S', True) + workdir = rd.getVar('WORKDIR', True) + + import bb.fetch + simplified = {} + src_uri = rd.getVar('SRC_URI', True).split() + for uri in src_uri: + if uri.endswith(';'): + uri = uri[:-1] + simple_uri = bb.fetch.URI(uri) + simple_uri.params = {} + simplified[str(simple_uri)] = uri + + copyfiles = {} + extralines = extralines or [] + for newfile, srcfile in files.iteritems(): + src_destdir = os.path.dirname(srcfile) + if not args.use_workdir: + if rd.getVar('S', True) == rd.getVar('STAGING_KERNEL_DIR', True): + srcdir = os.path.join(workdir, 'git') + if not bb.data.inherits_class('kernel-yocto', rd): + logger.warn('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git') + src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir) + src_destdir = os.path.normpath(src_destdir) + + source_uri = 'file://{0}'.format(os.path.basename(srcfile)) + if src_destdir and src_destdir != '.': + source_uri += ';subdir={0}'.format(src_destdir) + + simple = bb.fetch.URI(source_uri) + simple.params = {} + simple_str = str(simple) + if simple_str in simplified: + existing = simplified[simple_str] + if source_uri != existing: + logger.warn('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing)) + else: + logger.warn('{0!r} is already in SRC_URI, not adding'.format(source_uri)) + else: + extralines.append('SRC_URI += {0}'.format(source_uri)) + copyfiles[newfile] = srcfile + + oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines) + + +def appendsrcfiles(parser, args): + recipedata = _parse_recipe(args.recipe, tinfoil) + if not recipedata: + parser.error('RECIPE must be a valid recipe name') + + files = dict((f, os.path.join(args.destdir, os.path.basename(f))) + for f in args.files) + return appendsrc(args, files, recipedata) + + +def appendsrcfile(parser, args): + recipedata = _parse_recipe(args.recipe, tinfoil) + if not recipedata: + parser.error('RECIPE must be a valid recipe name') + + if not args.destfile: + args.destfile = os.path.basename(args.file) + elif args.destfile.endswith('/'): + args.destfile = os.path.join(args.destfile, os.path.basename(args.file)) + + return appendsrc(args, {args.file: args.destfile}, recipedata) + + +def layer(layerpath): + if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')): + raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath)) + return layerpath + + +def existing_path(filepath): + if not os.path.exists(filepath): + raise argparse.ArgumentTypeError('{0!r} must be an existing path'.format(filepath)) + return filepath + + +def existing_file(filepath): + filepath = existing_path(filepath) + if os.path.isdir(filepath): + raise argparse.ArgumentTypeError('{0!r} must be a file, not a directory'.format(filepath)) + return filepath + + +def destination_path(destpath): + if os.path.isabs(destpath): + raise argparse.ArgumentTypeError('{0!r} must be a relative path, not absolute'.format(destpath)) + return destpath + + +def target_path(targetpath): + if not os.path.isabs(targetpath): + raise argparse.ArgumentTypeError('{0!r} must be an absolute path, not relative'.format(targetpath)) + return targetpath + + +def register_commands(subparsers): + common = argparse.ArgumentParser(add_help=False) + common.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE') + common.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true') + common.add_argument('destlayer', metavar='DESTLAYER', help='Base directory of the destination layer to write the bbappend to', type=layer) + + parser_appendfile = subparsers.add_parser('appendfile', + parents=[common], + help='Create/update a bbappend to replace a target file', + description='Creates a bbappend (or updates an existing one) to replace the specified file that appears in the target system, determining the recipe that packages the file and the required path and name for the bbappend automatically. Note that the ability to determine the recipe packaging a particular file depends upon the recipe\'s do_packagedata task having already run prior to running this command (which it will have when the recipe has been built successfully, which in turn will have happened if one or more of the recipe\'s packages is included in an image that has been built successfully).') + parser_appendfile.add_argument('targetpath', help='Path to the file to be replaced (as it would appear within the target image, e.g. /etc/motd)', type=target_path) + parser_appendfile.add_argument('newfile', help='Custom file to replace the target file with', type=existing_file) + parser_appendfile.add_argument('-r', '--recipe', help='Override recipe to apply to (default is to find which recipe already packages the file)') + parser_appendfile.set_defaults(func=appendfile, parserecipes=True) + + common_src = argparse.ArgumentParser(add_help=False, parents=[common]) + common_src.add_argument('-W', '--workdir', help='Unpack file into WORKDIR rather than S', dest='use_workdir', action='store_true') + common_src.add_argument('recipe', metavar='RECIPE', help='Override recipe to apply to') + + parser = subparsers.add_parser('appendsrcfiles', + parents=[common_src], + help='Create/update a bbappend to add or replace source files', + description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.') + parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path) + parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path) + parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True) + + parser = subparsers.add_parser('appendsrcfile', + parents=[common_src], + help='Create/update a bbappend to add or replace a source file', + description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.') + parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path) + parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path) + parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True) diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py new file mode 100644 index 0000000..1f85fcf --- /dev/null +++ b/scripts/lib/recipetool/create.py @@ -0,0 +1,963 @@ +# Recipe creation tool - create command plugin +# +# Copyright (C) 2014-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import os +import argparse +import glob +import fnmatch +import re +import json +import logging +import scriptutils +import urlparse +import hashlib + +logger = logging.getLogger('recipetool') + +tinfoil = None +plugins = None + +def plugin_init(pluginlist): + # Take a reference to the list so we can use it later + global plugins + plugins = pluginlist + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + +class RecipeHandler(object): + recipelibmap = {} + recipeheadermap = {} + recipecmakefilemap = {} + recipebinmap = {} + + @staticmethod + def load_libmap(d): + '''Load library->recipe mapping''' + import oe.package + + if RecipeHandler.recipelibmap: + return + # First build up library->package mapping + shlib_providers = oe.package.read_shlib_providers(d) + libdir = d.getVar('libdir', True) + base_libdir = d.getVar('base_libdir', True) + libpaths = list(set([base_libdir, libdir])) + libname_re = re.compile('^lib(.+)\.so.*$') + pkglibmap = {} + for lib, item in shlib_providers.iteritems(): + for path, pkg in item.iteritems(): + if path in libpaths: + res = libname_re.match(lib) + if res: + libname = res.group(1) + if not libname in pkglibmap: + pkglibmap[libname] = pkg[0] + else: + logger.debug('unable to extract library name from %s' % lib) + + # Now turn it into a library->recipe mapping + pkgdata_dir = d.getVar('PKGDATA_DIR', True) + for libname, pkg in pkglibmap.iteritems(): + try: + with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: + for line in f: + if line.startswith('PN:'): + RecipeHandler.recipelibmap[libname] = line.split(':', 1)[-1].strip() + break + except IOError as ioe: + if ioe.errno == 2: + logger.warn('unable to find a pkgdata file for package %s' % pkg) + else: + raise + + # Some overrides - these should be mapped to the virtual + RecipeHandler.recipelibmap['GL'] = 'virtual/libgl' + RecipeHandler.recipelibmap['EGL'] = 'virtual/egl' + RecipeHandler.recipelibmap['GLESv2'] = 'virtual/libgles2' + + @staticmethod + def load_devel_filemap(d): + '''Build up development file->recipe mapping''' + if RecipeHandler.recipeheadermap: + return + pkgdata_dir = d.getVar('PKGDATA_DIR', True) + includedir = d.getVar('includedir', True) + cmakedir = os.path.join(d.getVar('libdir', True), 'cmake') + for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')): + with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: + pn = None + headers = [] + cmakefiles = [] + for line in f: + if line.startswith('PN:'): + pn = line.split(':', 1)[-1].strip() + elif line.startswith('FILES_INFO:'): + val = line.split(':', 1)[1].strip() + dictval = json.loads(val) + for fullpth in sorted(dictval): + if fullpth.startswith(includedir) and fullpth.endswith('.h'): + headers.append(os.path.relpath(fullpth, includedir)) + elif fullpth.startswith(cmakedir) and fullpth.endswith('.cmake'): + cmakefiles.append(os.path.relpath(fullpth, cmakedir)) + if pn and headers: + for header in headers: + RecipeHandler.recipeheadermap[header] = pn + if pn and cmakefiles: + for fn in cmakefiles: + RecipeHandler.recipecmakefilemap[fn] = pn + + @staticmethod + def load_binmap(d): + '''Build up native binary->recipe mapping''' + if RecipeHandler.recipebinmap: + return + sstate_manifests = d.getVar('SSTATE_MANIFESTS', True) + staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE', True) + build_arch = d.getVar('BUILD_ARCH', True) + fileprefix = 'manifest-%s-' % build_arch + for fn in glob.glob(os.path.join(sstate_manifests, '%s*-native.populate_sysroot' % fileprefix)): + with open(fn, 'r') as f: + pn = os.path.basename(fn).rsplit('.', 1)[0][len(fileprefix):] + for line in f: + if line.startswith(staging_bindir_native): + prog = os.path.basename(line.rstrip()) + RecipeHandler.recipebinmap[prog] = pn + + @staticmethod + def checkfiles(path, speclist, recursive=False): + results = [] + if recursive: + for root, _, files in os.walk(path): + for fn in files: + for spec in speclist: + if fnmatch.fnmatch(fn, spec): + results.append(os.path.join(root, fn)) + else: + for spec in speclist: + results.extend(glob.glob(os.path.join(path, spec))) + return results + + @staticmethod + def handle_depends(libdeps, pcdeps, deps, outlines, values, d): + if pcdeps: + recipemap = read_pkgconfig_provides(d) + if libdeps: + RecipeHandler.load_libmap(d) + + ignorelibs = ['socket'] + ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native', 'coreutils-native'] + + unmappedpc = [] + pcdeps = list(set(pcdeps)) + for pcdep in pcdeps: + if isinstance(pcdep, basestring): + recipe = recipemap.get(pcdep, None) + if recipe: + deps.append(recipe) + else: + if not pcdep.startswith('$'): + unmappedpc.append(pcdep) + else: + for item in pcdep: + recipe = recipemap.get(pcdep, None) + if recipe: + deps.append(recipe) + break + else: + unmappedpc.append('(%s)' % ' or '.join(pcdep)) + + unmappedlibs = [] + for libdep in libdeps: + if isinstance(libdep, tuple): + lib, header = libdep + else: + lib = libdep + header = None + + if lib in ignorelibs: + logger.debug('Ignoring library dependency %s' % lib) + continue + + recipe = RecipeHandler.recipelibmap.get(lib, None) + if recipe: + deps.append(recipe) + elif recipe is None: + if header: + RecipeHandler.load_devel_filemap(d) + recipe = RecipeHandler.recipeheadermap.get(header, None) + if recipe: + deps.append(recipe) + elif recipe is None: + unmappedlibs.append(lib) + else: + unmappedlibs.append(lib) + + deps = set(deps).difference(set(ignoredeps)) + + if unmappedpc: + outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmappedpc)) + outlines.append('# (this is based on recipes that have previously been built and packaged)') + + if unmappedlibs: + outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs)))) + outlines.append('# (this is based on recipes that have previously been built and packaged)') + + if deps: + values['DEPENDS'] = ' '.join(deps) + + def genfunction(self, outlines, funcname, content, python=False, forcespace=False): + if python: + prefix = 'python ' + else: + prefix = '' + outlines.append('%s%s () {' % (prefix, funcname)) + if python or forcespace: + indent = ' ' + else: + indent = '\t' + addnoop = not python + for line in content: + outlines.append('%s%s' % (indent, line)) + if addnoop: + strippedline = line.lstrip() + if strippedline and not strippedline.startswith('#'): + addnoop = False + if addnoop: + # Without this there'll be a syntax error + outlines.append('%s:' % indent) + outlines.append('}') + outlines.append('') + + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + return False + + +def validate_pv(pv): + if not pv or '_version' in pv.lower() or pv[0] not in '0123456789': + return False + return True + +def determine_from_filename(srcfile): + """Determine name and version from a filename""" + part = '' + if '.tar.' in srcfile: + namepart = srcfile.split('.tar.')[0].lower() + else: + namepart = os.path.splitext(srcfile)[0].lower() + splitval = namepart.rsplit('_', 1) + if len(splitval) == 1: + splitval = namepart.rsplit('-', 1) + pn = splitval[0].replace('_', '-') + if len(splitval) > 1: + if splitval[1][0] in '0123456789': + pv = splitval[1] + else: + pn = '-'.join(splitval).replace('_', '-') + pv = None + else: + pv = None + return (pn, pv) + +def determine_from_url(srcuri): + """Determine name and version from a URL""" + pn = None + pv = None + parseres = urlparse.urlparse(srcuri.lower().split(';', 1)[0]) + if parseres.path: + if 'github.com' in parseres.netloc: + res = re.search(r'.*/(.*?)/archive/(.*)-final\.(tar|zip)', parseres.path) + if res: + pn = res.group(1).strip().replace('_', '-') + pv = res.group(2).strip().replace('_', '.') + else: + res = re.search(r'.*/(.*?)/archive/v?(.*)\.(tar|zip)', parseres.path) + if res: + pn = res.group(1).strip().replace('_', '-') + pv = res.group(2).strip().replace('_', '.') + elif 'bitbucket.org' in parseres.netloc: + res = re.search(r'.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9a-zA-Z_.]*)\.(tar|zip)', parseres.path) + if res: + pn = res.group(1).strip().replace('_', '-') + pv = res.group(2).strip().replace('_', '.') + + if not pn and not pv: + srcfile = os.path.basename(parseres.path.rstrip('/')) + pn, pv = determine_from_filename(srcfile) + + logger.debug('Determined from source URL: name = "%s", version = "%s"' % (pn, pv)) + return (pn, pv) + +def supports_srcrev(uri): + localdata = bb.data.createCopy(tinfoil.config_data) + # This is a bit sad, but if you don't have this set there can be some + # odd interactions with the urldata cache which lead to errors + localdata.setVar('SRCREV', '${AUTOREV}') + bb.data.update_data(localdata) + fetcher = bb.fetch2.Fetch([uri], localdata) + urldata = fetcher.ud + for u in urldata: + if urldata[u].method.supports_srcrev(): + return True + return False + +def reformat_git_uri(uri): + '''Convert any http[s]://....git URI into git://...;protocol=http[s]''' + checkuri = uri.split(';', 1)[0] + if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri): + res = re.match('(https?)://([^;]+(\.git)?)(;.*)?$', uri) + if res: + # Need to switch the URI around so that the git fetcher is used + return 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(4) or '') + return uri + +def create_recipe(args): + import bb.process + import tempfile + import shutil + + pkgarch = "" + if args.machine: + pkgarch = "${MACHINE_ARCH}" + + checksums = (None, None) + tempsrc = '' + srcsubdir = '' + srcrev = '${AUTOREV}' + if '://' in args.source: + # Fetch a URL + fetchuri = reformat_git_uri(urlparse.urldefrag(args.source)[0]) + if args.binary: + # Assume the archive contains the directory structure verbatim + # so we need to extract to a subdirectory + fetchuri += ';subdir=%s' % os.path.splitext(os.path.basename(urlparse.urlsplit(fetchuri).path))[0] + srcuri = fetchuri + rev_re = re.compile(';rev=([^;]+)') + res = rev_re.search(srcuri) + if res: + srcrev = res.group(1) + srcuri = rev_re.sub('', srcuri) + tempsrc = tempfile.mkdtemp(prefix='recipetool-') + srctree = tempsrc + if fetchuri.startswith('npm://'): + # Check if npm is available + npm = bb.utils.which(tinfoil.config_data.getVar('PATH', True), 'npm') + if not npm: + logger.error('npm:// URL requested but npm is not available - you need to either build nodejs-native or install npm using your package manager') + sys.exit(1) + logger.info('Fetching %s...' % srcuri) + try: + checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) + except bb.fetch2.BBFetchException as e: + logger.error(str(e).rstrip()) + sys.exit(1) + dirlist = os.listdir(srctree) + if 'git.indirectionsymlink' in dirlist: + dirlist.remove('git.indirectionsymlink') + if len(dirlist) == 1: + singleitem = os.path.join(srctree, dirlist[0]) + if os.path.isdir(singleitem): + # We unpacked a single directory, so we should use that + srcsubdir = dirlist[0] + srctree = os.path.join(srctree, srcsubdir) + else: + with open(singleitem, 'r') as f: + if '<html' in f.read(100).lower(): + logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri) + sys.exit(1) + else: + # Assume we're pointing to an existing source tree + if args.extract_to: + logger.error('--extract-to cannot be specified if source is a directory') + sys.exit(1) + if not os.path.isdir(args.source): + logger.error('Invalid source directory %s' % args.source) + sys.exit(1) + srctree = args.source + srcuri = '' + if os.path.exists(os.path.join(srctree, '.git')): + # Try to get upstream repo location from origin remote + try: + stdout, _ = bb.process.run('git remote -v', cwd=srctree, shell=True) + except bb.process.ExecutionError as e: + stdout = None + if stdout: + for line in stdout.splitlines(): + splitline = line.split() + if len(splitline) > 1: + if splitline[0] == 'origin' and '://' in splitline[1]: + srcuri = reformat_git_uri(splitline[1]) + srcsubdir = 'git' + break + + if args.src_subdir: + srcsubdir = os.path.join(srcsubdir, args.src_subdir) + srctree_use = os.path.join(srctree, args.src_subdir) + else: + srctree_use = srctree + + if args.outfile and os.path.isdir(args.outfile): + outfile = None + outdir = args.outfile + else: + outfile = args.outfile + outdir = None + if outfile and outfile != '-': + if os.path.exists(outfile): + logger.error('Output file %s already exists' % outfile) + sys.exit(1) + + lines_before = [] + lines_after = [] + + lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) + lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.') + lines_before.append('# (Feel free to remove these comments when editing.)') + lines_before.append('#') + + licvalues = guess_license(srctree_use) + lic_files_chksum = [] + if licvalues: + licenses = [] + for licvalue in licvalues: + if not licvalue[0] in licenses: + licenses.append(licvalue[0]) + lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) + lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is') + lines_before.append('# your responsibility to verify that the values are complete and correct.') + if len(licvalues) > 1: + lines_before.append('#') + lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate') + lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there') + lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying') + lines_before.append('# documentation to determine which situation is applicable.') + else: + lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying') + lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.') + lines_before.append('#') + lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if') + lines_before.append('# this is not accurate with respect to the licensing of the software being built (it') + lines_before.append('# will not be in most cases) you must specify the correct value before using this') + lines_before.append('# recipe for anything other than initial testing/development!') + licenses = ['CLOSED'] + lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) + lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) + lines_before.append('') + + classes = [] + + # FIXME This is kind of a hack, we probably ought to be using bitbake to do this + pn = None + pv = None + if outfile: + recipefn = os.path.splitext(os.path.basename(outfile))[0] + fnsplit = recipefn.split('_') + if len(fnsplit) > 1: + pn = fnsplit[0] + pv = fnsplit[1] + else: + pn = recipefn + + if args.version: + pv = args.version + + if args.name: + pn = args.name + if args.name.endswith('-native'): + if args.also_native: + logger.error('--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native') + sys.exit(1) + classes.append('native') + elif args.name.startswith('nativesdk-'): + if args.also_native: + logger.error('--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)') + sys.exit(1) + classes.append('nativesdk') + + if pv and pv not in 'git svn hg'.split(): + realpv = pv + else: + realpv = None + + if srcuri and not realpv or not pn: + name_pn, name_pv = determine_from_url(srcuri) + if name_pn and not pn: + pn = name_pn + if name_pv and not realpv: + realpv = name_pv + + + if not srcuri: + lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') + lines_before.append('SRC_URI = "%s"' % srcuri) + (md5value, sha256value) = checksums + if md5value: + lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) + if sha256value: + lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) + if srcuri and supports_srcrev(srcuri): + lines_before.append('') + lines_before.append('# Modify these as desired') + lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) + lines_before.append('SRCREV = "%s"' % srcrev) + lines_before.append('') + + if srcsubdir: + lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) + lines_before.append('') + + if pkgarch: + lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) + lines_after.append('') + + if args.binary: + lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') + lines_after.append('') + + # Find all plugins that want to register handlers + logger.debug('Loading recipe handlers') + raw_handlers = [] + for plugin in plugins: + if hasattr(plugin, 'register_recipe_handlers'): + plugin.register_recipe_handlers(raw_handlers) + # Sort handlers by priority + handlers = [] + for i, handler in enumerate(raw_handlers): + if isinstance(handler, tuple): + handlers.append((handler[0], handler[1], i)) + else: + handlers.append((handler, 0, i)) + handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) + for handler, priority, _ in handlers: + logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) + handlers = [item[0] for item in handlers] + + # Apply the handlers + handled = [] + handled.append(('license', licvalues)) + + if args.binary: + classes.append('bin_package') + handled.append('buildsystem') + + extravalues = {} + for handler in handlers: + handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) + + extrafiles = extravalues.pop('extrafiles', {}) + + if not realpv: + realpv = extravalues.get('PV', None) + if realpv: + if not validate_pv(realpv): + realpv = None + else: + realpv = realpv.lower().split()[0] + if '_' in realpv: + realpv = realpv.replace('_', '-') + if not pn: + pn = extravalues.get('PN', None) + if pn: + if pn.startswith('GNU '): + pn = pn[4:] + if ' ' in pn: + # Probably a descriptive identifier rather than a proper name + pn = None + else: + pn = pn.lower() + if '_' in pn: + pn = pn.replace('_', '-') + + if not outfile: + if not pn: + logger.error('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile') + # devtool looks for this specific exit code, so don't change it + sys.exit(15) + else: + if srcuri and srcuri.startswith(('git://', 'hg://', 'svn://')): + outfile = '%s_%s.bb' % (pn, srcuri.split(':', 1)[0]) + elif realpv: + outfile = '%s_%s.bb' % (pn, realpv) + else: + outfile = '%s.bb' % pn + if outdir: + outfile = os.path.join(outdir, outfile) + # We need to check this again + if os.path.exists(outfile): + logger.error('Output file %s already exists' % outfile) + sys.exit(1) + + # Move any extra files the plugins created to a directory next to the recipe + if extrafiles: + if outfile == '-': + extraoutdir = pn + else: + extraoutdir = os.path.join(os.path.dirname(outfile), pn) + bb.utils.mkdirhier(extraoutdir) + for destfn, extrafile in extrafiles.iteritems(): + shutil.move(extrafile, os.path.join(extraoutdir, destfn)) + + lines = lines_before + lines_before = [] + skipblank = True + for line in lines: + if skipblank: + skipblank = False + if not line: + continue + if line.startswith('S = '): + if realpv and pv not in 'git svn hg'.split(): + line = line.replace(realpv, '${PV}') + if pn: + line = line.replace(pn, '${BPN}') + if line == 'S = "${WORKDIR}/${BPN}-${PV}"': + skipblank = True + continue + elif line.startswith('SRC_URI = '): + if realpv: + line = line.replace(realpv, '${PV}') + elif line.startswith('PV = '): + if realpv: + line = re.sub('"[^+]*\+', '"%s+' % realpv, line) + lines_before.append(line) + + if args.also_native: + lines = lines_after + lines_after = [] + bbclassextend = None + for line in lines: + if line.startswith('BBCLASSEXTEND ='): + splitval = line.split('"') + if len(splitval) > 1: + bbclassextend = splitval[1].split() + if not 'native' in bbclassextend: + bbclassextend.insert(0, 'native') + line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend) + lines_after.append(line) + if not bbclassextend: + lines_after.append('BBCLASSEXTEND = "native"') + + outlines = [] + outlines.extend(lines_before) + if classes: + if outlines[-1] and not outlines[-1].startswith('#'): + outlines.append('') + outlines.append('inherit %s' % ' '.join(classes)) + outlines.append('') + outlines.extend(lines_after) + + if args.extract_to: + scriptutils.git_convert_standalone_clone(srctree) + if os.path.isdir(args.extract_to): + # If the directory exists we'll move the temp dir into it instead of + # its contents - of course, we could try to always move its contents + # but that is a pain if there are symlinks; the simplest solution is + # to just remove it first + os.rmdir(args.extract_to) + shutil.move(srctree, args.extract_to) + if tempsrc == srctree: + tempsrc = None + logger.info('Source extracted to %s' % args.extract_to) + + if outfile == '-': + sys.stdout.write('\n'.join(outlines) + '\n') + else: + with open(outfile, 'w') as f: + f.write('\n'.join(outlines) + '\n') + logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile) + + if tempsrc: + shutil.rmtree(tempsrc) + + return 0 + +def get_license_md5sums(d, static_only=False): + import bb.utils + md5sums = {} + if not static_only: + # Gather md5sums of license files in common license dir + commonlicdir = d.getVar('COMMON_LICENSE_DIR', True) + for fn in os.listdir(commonlicdir): + md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn)) + md5sums[md5value] = fn + # The following were extracted from common values in various recipes + # (double checking the license against the license file itself, not just + # the LICENSE value in the recipe) + md5sums['94d55d512a9ba36caa9b7df079bae19f'] = 'GPLv2' + md5sums['b234ee4d69f5fce4486a80fdaf4a4263'] = 'GPLv2' + md5sums['59530bdf33659b29e73d4adb9f9f6552'] = 'GPLv2' + md5sums['0636e73ff0215e8d672dc4c32c317bb3'] = 'GPLv2' + md5sums['eb723b61539feef013de476e68b5c50a'] = 'GPLv2' + md5sums['751419260aa954499f7abaabaa882bbe'] = 'GPLv2' + md5sums['393a5ca445f6965873eca0259a17f833'] = 'GPLv2' + md5sums['12f884d2ae1ff87c09e5b7ccc2c4ca7e'] = 'GPLv2' + md5sums['8ca43cbc842c2336e835926c2166c28b'] = 'GPLv2' + md5sums['ebb5c50ab7cab4baeffba14977030c07'] = 'GPLv2' + md5sums['c93c0550bd3173f4504b2cbd8991e50b'] = 'GPLv2' + md5sums['9ac2e7cff1ddaf48b6eab6028f23ef88'] = 'GPLv2' + md5sums['4325afd396febcb659c36b49533135d4'] = 'GPLv2' + md5sums['18810669f13b87348459e611d31ab760'] = 'GPLv2' + md5sums['d7810fab7487fb0aad327b76f1be7cd7'] = 'GPLv2' # the Linux kernel's COPYING file + md5sums['bbb461211a33b134d42ed5ee802b37ff'] = 'LGPLv2.1' + md5sums['7fbc338309ac38fefcd64b04bb903e34'] = 'LGPLv2.1' + md5sums['4fbd65380cdd255951079008b364516c'] = 'LGPLv2.1' + md5sums['2d5025d4aa3495befef8f17206a5b0a1'] = 'LGPLv2.1' + md5sums['fbc093901857fcd118f065f900982c24'] = 'LGPLv2.1' + md5sums['a6f89e2100d9b6cdffcea4f398e37343'] = 'LGPLv2.1' + md5sums['d8045f3b8f929c1cb29a1e3fd737b499'] = 'LGPLv2.1' + md5sums['fad9b3332be894bab9bc501572864b29'] = 'LGPLv2.1' + md5sums['3bf50002aefd002f49e7bb854063f7e7'] = 'LGPLv2' + md5sums['9f604d8a4f8e74f4f5140845a21b6674'] = 'LGPLv2' + md5sums['5f30f0716dfdd0d91eb439ebec522ec2'] = 'LGPLv2' + md5sums['55ca817ccb7d5b5b66355690e9abc605'] = 'LGPLv2' + md5sums['252890d9eee26aab7b432e8b8a616475'] = 'LGPLv2' + md5sums['3214f080875748938ba060314b4f727d'] = 'LGPLv2' + md5sums['db979804f025cf55aabec7129cb671ed'] = 'LGPLv2' + md5sums['d32239bcb673463ab874e80d47fae504'] = 'GPLv3' + md5sums['f27defe1e96c2e1ecd4e0c9be8967949'] = 'GPLv3' + md5sums['6a6a8e020838b23406c81b19c1d46df6'] = 'LGPLv3' + md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0' + md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules + md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause' + return md5sums + +def crunch_license(licfile): + ''' + Remove non-material text from a license file and then check + its md5sum against a known list. This works well for licenses + which contain a copyright statement, but is also a useful way + to handle people's insistence upon reformatting the license text + slightly (with no material difference to the text of the + license). + ''' + + import oe.utils + + # Note: these are carefully constructed! + license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$') + license_statement_re = re.compile('^This (project|software) is( free software)? released under the .{1,10} [Ll]icen[sc]e:?$') + copyright_re = re.compile('^(#+)? *Copyright .*$') + + crunched_md5sums = {} + # The following two were gleaned from the "forever" npm package + crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC' + crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT' + # https://github.com/vasi/pixz/blob/master/LICENSE + crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause' + # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt + crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause' + # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE + crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2' + # https://github.com/datto/dattobd/blob/master/COPYING + # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT + crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2' + # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt + # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD + crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2' + # https://github.com/gkos/nrf24/blob/master/COPYING + crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2' + # https://github.com/josch09/resetusb/blob/master/COPYING + crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3' + # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1 + crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1' + # unixODBC-2.3.4 COPYING + crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1' + # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3 + crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3' + lictext = [] + with open(licfile, 'r') as f: + for line in f: + # Drop opening statements + if copyright_re.match(line): + continue + elif license_title_re.match(line): + continue + elif license_statement_re.match(line): + continue + # Squash spaces, and replace smart quotes, double quotes + # and backticks with single quotes + line = oe.utils.squashspaces(line.strip()).decode("utf-8") + line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'') + if line: + lictext.append(line) + + m = hashlib.md5() + try: + m.update(' '.join(lictext)) + md5val = m.hexdigest() + except UnicodeEncodeError: + md5val = None + lictext = '' + license = crunched_md5sums.get(md5val, None) + return license, md5val, lictext + +def guess_license(srctree): + import bb + md5sums = get_license_md5sums(tinfoil.config_data) + + licenses = [] + licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*'] + licfiles = [] + for root, dirs, files in os.walk(srctree): + for fn in files: + for spec in licspecs: + if fnmatch.fnmatch(fn, spec): + fullpath = os.path.join(root, fn) + if not fullpath in licfiles: + licfiles.append(fullpath) + for licfile in licfiles: + md5value = bb.utils.md5_file(licfile) + license = md5sums.get(md5value, None) + if not license: + license, crunched_md5, lictext = crunch_license(licfile) + if not license: + license = 'Unknown' + licenses.append((license, os.path.relpath(licfile, srctree), md5value)) + + # FIXME should we grab at least one source file with a license header and add that too? + + return licenses + +def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'): + """ + Given a list of (license, path, md5sum) as returned by guess_license(), + a dict of package name to path mappings, write out a set of + package-specific LICENSE values. + """ + pkglicenses = {pn: []} + for license, licpath, _ in licvalues: + for pkgname, pkgpath in packages.iteritems(): + if licpath.startswith(pkgpath + '/'): + if pkgname in pkglicenses: + pkglicenses[pkgname].append(license) + else: + pkglicenses[pkgname] = [license] + break + else: + # Accumulate on the main package + pkglicenses[pn].append(license) + outlicenses = {} + for pkgname in packages: + license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown' + if license == 'Unknown' and pkgname in fallback_licenses: + license = fallback_licenses[pkgname] + outlines.append('LICENSE_%s = "%s"' % (pkgname, license)) + outlicenses[pkgname] = license.split() + return outlicenses + +def read_pkgconfig_provides(d): + pkgdatadir = d.getVar('PKGDATA_DIR', True) + pkgmap = {} + for fn in glob.glob(os.path.join(pkgdatadir, 'shlibs2', '*.pclist')): + with open(fn, 'r') as f: + for line in f: + pkgmap[os.path.basename(line.rstrip())] = os.path.splitext(os.path.basename(fn))[0] + recipemap = {} + for pc, pkg in pkgmap.iteritems(): + pkgdatafile = os.path.join(pkgdatadir, 'runtime', pkg) + if os.path.exists(pkgdatafile): + with open(pkgdatafile, 'r') as f: + for line in f: + if line.startswith('PN: '): + recipemap[pc] = line.split(':', 1)[1].strip() + return recipemap + +def convert_pkginfo(pkginfofile): + values = {} + with open(pkginfofile, 'r') as f: + indesc = False + for line in f: + if indesc: + if line.strip(): + values['DESCRIPTION'] += ' ' + line.strip() + else: + indesc = False + else: + splitline = line.split(': ', 1) + key = line[0] + value = line[1] + if key == 'LICENSE': + for dep in value.split(','): + dep = dep.split()[0] + mapped = depmap.get(dep, '') + if mapped: + depends.append(mapped) + elif key == 'License': + values['LICENSE'] = value + elif key == 'Summary': + values['SUMMARY'] = value + elif key == 'Description': + values['DESCRIPTION'] = value + indesc = True + return values + +def convert_debian(debpath): + # FIXME extend this mapping - perhaps use distro_alias.inc? + depmap = {'libz-dev': 'zlib'} + + values = {} + depends = [] + with open(os.path.join(debpath, 'control')) as f: + indesc = False + for line in f: + if indesc: + if line.strip(): + if line.startswith(' This package contains'): + indesc = False + else: + values['DESCRIPTION'] += ' ' + line.strip() + else: + indesc = False + else: + splitline = line.split(':', 1) + key = line[0] + value = line[1] + if key == 'Build-Depends': + for dep in value.split(','): + dep = dep.split()[0] + mapped = depmap.get(dep, '') + if mapped: + depends.append(mapped) + elif key == 'Section': + values['SECTION'] = value + elif key == 'Description': + values['SUMMARY'] = value + indesc = True + + if depends: + values['DEPENDS'] = ' '.join(depends) + + return values + + +def register_commands(subparsers): + parser_create = subparsers.add_parser('create', + help='Create a new recipe', + description='Creates a new recipe from a source tree') + parser_create.add_argument('source', help='Path or URL to source') + parser_create.add_argument('-o', '--outfile', help='Specify filename for recipe to create') + parser_create.add_argument('-m', '--machine', help='Make recipe machine-specific as opposed to architecture-specific', action='store_true') + parser_create.add_argument('-x', '--extract-to', metavar='EXTRACTPATH', help='Assuming source is a URL, fetch it and extract it to the directory specified as %(metavar)s') + parser_create.add_argument('-N', '--name', help='Name to use within recipe (PN)') + parser_create.add_argument('-V', '--version', help='Version to use within recipe (PV)') + parser_create.add_argument('-b', '--binary', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure)', action='store_true') + parser_create.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') + parser_create.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') + parser_create.set_defaults(func=create_recipe) + diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py new file mode 100644 index 0000000..f84ec3d --- /dev/null +++ b/scripts/lib/recipetool/create_buildsys.py @@ -0,0 +1,859 @@ +# Recipe creation tool - create command build system handlers +# +# Copyright (C) 2014-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import re +import logging +import glob +from recipetool.create import RecipeHandler, validate_pv + +logger = logging.getLogger('recipetool') + +tinfoil = None +plugins = None + +def plugin_init(pluginlist): + # Take a reference to the list so we can use it later + global plugins + plugins = pluginlist + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +class CmakeRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'buildsystem' in handled: + return False + + if RecipeHandler.checkfiles(srctree, ['CMakeLists.txt']): + classes.append('cmake') + values = CmakeRecipeHandler.extract_cmake_deps(lines_before, srctree, extravalues) + classes.extend(values.pop('inherit', '').split()) + for var, value in values.iteritems(): + lines_before.append('%s = "%s"' % (var, value)) + lines_after.append('# Specify any options you want to pass to cmake using EXTRA_OECMAKE:') + lines_after.append('EXTRA_OECMAKE = ""') + lines_after.append('') + handled.append('buildsystem') + return True + return False + + @staticmethod + def extract_cmake_deps(outlines, srctree, extravalues, cmakelistsfile=None): + # Find all plugins that want to register handlers + logger.debug('Loading cmake handlers') + handlers = [] + for plugin in plugins: + if hasattr(plugin, 'register_cmake_handlers'): + plugin.register_cmake_handlers(handlers) + + values = {} + inherits = [] + + if cmakelistsfile: + srcfiles = [cmakelistsfile] + else: + srcfiles = RecipeHandler.checkfiles(srctree, ['CMakeLists.txt']) + + # Note that some of these are non-standard, but probably better to + # be able to map them anyway if we see them + cmake_pkgmap = {'alsa': 'alsa-lib', + 'aspell': 'aspell', + 'atk': 'atk', + 'bison': 'bison-native', + 'boost': 'boost', + 'bzip2': 'bzip2', + 'cairo': 'cairo', + 'cups': 'cups', + 'curl': 'curl', + 'curses': 'ncurses', + 'cvs': 'cvs', + 'drm': 'libdrm', + 'dbus': 'dbus', + 'dbusglib': 'dbus-glib', + 'egl': 'virtual/egl', + 'expat': 'expat', + 'flex': 'flex-native', + 'fontconfig': 'fontconfig', + 'freetype': 'freetype', + 'gettext': '', + 'git': '', + 'gio': 'glib-2.0', + 'giounix': 'glib-2.0', + 'glew': 'glew', + 'glib': 'glib-2.0', + 'glib2': 'glib-2.0', + 'glu': 'libglu', + 'glut': 'freeglut', + 'gobject': 'glib-2.0', + 'gperf': 'gperf-native', + 'gnutls': 'gnutls', + 'gtk2': 'gtk+', + 'gtk3': 'gtk+3', + 'gtk': 'gtk+3', + 'harfbuzz': 'harfbuzz', + 'icu': 'icu', + 'intl': 'virtual/libintl', + 'jpeg': 'jpeg', + 'libarchive': 'libarchive', + 'libiconv': 'virtual/libiconv', + 'liblzma': 'xz', + 'libxml2': 'libxml2', + 'libxslt': 'libxslt', + 'opengl': 'virtual/libgl', + 'openmp': '', + 'openssl': 'openssl', + 'pango': 'pango', + 'perl': '', + 'perllibs': '', + 'pkgconfig': '', + 'png': 'libpng', + 'pthread': '', + 'pythoninterp': '', + 'pythonlibs': '', + 'ruby': 'ruby-native', + 'sdl': 'libsdl', + 'sdl2': 'libsdl2', + 'subversion': 'subversion-native', + 'swig': 'swig-native', + 'tcl': 'tcl-native', + 'threads': '', + 'tiff': 'tiff', + 'wget': 'wget', + 'x11': 'libx11', + 'xcb': 'libxcb', + 'xext': 'libxext', + 'xfixes': 'libxfixes', + 'zlib': 'zlib', + } + + pcdeps = [] + libdeps = [] + deps = [] + unmappedpkgs = [] + + proj_re = re.compile('project\s*\(([^)]*)\)', re.IGNORECASE) + pkgcm_re = re.compile('pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE) + pkgsm_re = re.compile('pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE) + findpackage_re = re.compile('find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE) + findlibrary_re = re.compile('find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*') + checklib_re = re.compile('check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE) + include_re = re.compile('include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE) + subdir_re = re.compile('add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE) + dep_re = re.compile('([^ ><=]+)( *[<>=]+ *[^ ><=]+)?') + + def find_cmake_package(pkg): + RecipeHandler.load_devel_filemap(tinfoil.config_data) + for fn, pn in RecipeHandler.recipecmakefilemap.iteritems(): + splitname = fn.split('/') + if len(splitname) > 1: + if splitname[0].lower().startswith(pkg.lower()): + if splitname[1] == '%s-config.cmake' % pkg.lower() or splitname[1] == '%sConfig.cmake' % pkg or splitname[1] == 'Find%s.cmake' % pkg: + return pn + return None + + def interpret_value(value): + return value.strip('"') + + def parse_cmake_file(fn, paths=None): + searchpaths = (paths or []) + [os.path.dirname(fn)] + logger.debug('Parsing file %s' % fn) + with open(fn, 'r') as f: + for line in f: + line = line.strip() + for handler in handlers: + if handler.process_line(srctree, fn, line, libdeps, pcdeps, deps, outlines, inherits, values): + continue + res = include_re.match(line) + if res: + includefn = bb.utils.which(':'.join(searchpaths), res.group(1)) + if includefn: + parse_cmake_file(includefn, searchpaths) + else: + logger.debug('Unable to recurse into include file %s' % res.group(1)) + continue + res = subdir_re.match(line) + if res: + subdirfn = os.path.join(os.path.dirname(fn), res.group(1), 'CMakeLists.txt') + if os.path.exists(subdirfn): + parse_cmake_file(subdirfn, searchpaths) + else: + logger.debug('Unable to recurse into subdirectory file %s' % subdirfn) + continue + res = proj_re.match(line) + if res: + extravalues['PN'] = interpret_value(res.group(1).split()[0]) + continue + res = pkgcm_re.match(line) + if res: + res = dep_re.findall(res.group(2)) + if res: + pcdeps.extend([interpret_value(x[0]) for x in res]) + inherits.append('pkgconfig') + continue + res = pkgsm_re.match(line) + if res: + res = dep_re.findall(res.group(2)) + if res: + # Note: appending a tuple here! + item = tuple((interpret_value(x[0]) for x in res)) + if len(item) == 1: + item = item[0] + pcdeps.append(item) + inherits.append('pkgconfig') + continue + res = findpackage_re.match(line) + if res: + origpkg = res.group(1) + pkg = interpret_value(origpkg) + found = False + for handler in handlers: + if handler.process_findpackage(srctree, fn, pkg, deps, outlines, inherits, values): + logger.debug('Mapped CMake package %s via handler %s' % (pkg, handler.__class__.__name__)) + found = True + break + if found: + continue + elif pkg == 'Gettext': + inherits.append('gettext') + elif pkg == 'Perl': + inherits.append('perlnative') + elif pkg == 'PkgConfig': + inherits.append('pkgconfig') + elif pkg == 'PythonInterp': + inherits.append('pythonnative') + elif pkg == 'PythonLibs': + inherits.append('python-dir') + else: + # Try to map via looking at installed CMake packages in pkgdata + dep = find_cmake_package(pkg) + if dep: + logger.debug('Mapped CMake package %s to recipe %s via pkgdata' % (pkg, dep)) + deps.append(dep) + else: + dep = cmake_pkgmap.get(pkg.lower(), None) + if dep: + logger.debug('Mapped CMake package %s to recipe %s via internal list' % (pkg, dep)) + deps.append(dep) + elif dep is None: + unmappedpkgs.append(origpkg) + continue + res = checklib_re.match(line) + if res: + lib = interpret_value(res.group(1)) + if not lib.startswith('$'): + libdeps.append(lib) + res = findlibrary_re.match(line) + if res: + libs = res.group(2).split() + for lib in libs: + if lib in ['HINTS', 'PATHS', 'PATH_SUFFIXES', 'DOC', 'NAMES_PER_DIR'] or lib.startswith(('NO_', 'CMAKE_', 'ONLY_CMAKE_')): + break + lib = interpret_value(lib) + if not lib.startswith('$'): + libdeps.append(lib) + if line.lower().startswith('useswig'): + deps.append('swig-native') + continue + + parse_cmake_file(srcfiles[0]) + + if unmappedpkgs: + outlines.append('# NOTE: unable to map the following CMake package dependencies: %s' % ' '.join(list(set(unmappedpkgs)))) + + RecipeHandler.handle_depends(libdeps, pcdeps, deps, outlines, values, tinfoil.config_data) + + for handler in handlers: + handler.post_process(srctree, libdeps, pcdeps, deps, outlines, inherits, values) + + if inherits: + values['inherit'] = ' '.join(list(set(inherits))) + + return values + + +class CmakeExtensionHandler(object): + '''Base class for CMake extension handlers''' + def process_line(self, srctree, fn, line, libdeps, pcdeps, deps, outlines, inherits, values): + ''' + Handle a line parsed out of an CMake file. + Return True if you've completely handled the passed in line, otherwise return False. + ''' + return False + + def process_findpackage(self, srctree, fn, pkg, deps, outlines, inherits, values): + ''' + Handle a find_package package parsed out of a CMake file. + Return True if you've completely handled the passed in package, otherwise return False. + ''' + return False + + def post_process(self, srctree, fn, pkg, deps, outlines, inherits, values): + ''' + Apply any desired post-processing on the output + ''' + return + + + +class SconsRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'buildsystem' in handled: + return False + + if RecipeHandler.checkfiles(srctree, ['SConstruct', 'Sconstruct', 'sconstruct']): + classes.append('scons') + lines_after.append('# Specify any options you want to pass to scons using EXTRA_OESCONS:') + lines_after.append('EXTRA_OESCONS = ""') + lines_after.append('') + handled.append('buildsystem') + return True + return False + + +class QmakeRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'buildsystem' in handled: + return False + + if RecipeHandler.checkfiles(srctree, ['*.pro']): + classes.append('qmake2') + handled.append('buildsystem') + return True + return False + + +class AutotoolsRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'buildsystem' in handled: + return False + + autoconf = False + if RecipeHandler.checkfiles(srctree, ['configure.ac', 'configure.in']): + autoconf = True + values = AutotoolsRecipeHandler.extract_autotools_deps(lines_before, srctree, extravalues) + classes.extend(values.pop('inherit', '').split()) + for var, value in values.iteritems(): + lines_before.append('%s = "%s"' % (var, value)) + else: + conffile = RecipeHandler.checkfiles(srctree, ['configure']) + if conffile: + # Check if this is just a pre-generated autoconf configure script + with open(conffile[0], 'r') as f: + for i in range(1, 10): + if 'Generated by GNU Autoconf' in f.readline(): + autoconf = True + break + + if autoconf and not ('PV' in extravalues and 'PN' in extravalues): + # Last resort + conffile = RecipeHandler.checkfiles(srctree, ['configure']) + if conffile: + with open(conffile[0], 'r') as f: + for line in f: + line = line.strip() + if line.startswith('VERSION=') or line.startswith('PACKAGE_VERSION='): + pv = line.split('=')[1].strip('"\'') + if pv and not 'PV' in extravalues and validate_pv(pv): + extravalues['PV'] = pv + elif line.startswith('PACKAGE_NAME=') or line.startswith('PACKAGE='): + pn = line.split('=')[1].strip('"\'') + if pn and not 'PN' in extravalues: + extravalues['PN'] = pn + + if autoconf: + lines_before.append('') + lines_before.append('# NOTE: if this software is not capable of being built in a separate build directory') + lines_before.append('# from the source, you should replace autotools with autotools-brokensep in the') + lines_before.append('# inherit line') + classes.append('autotools') + lines_after.append('# Specify any options you want to pass to the configure script using EXTRA_OECONF:') + lines_after.append('EXTRA_OECONF = ""') + lines_after.append('') + handled.append('buildsystem') + return True + + return False + + @staticmethod + def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None): + import shlex + + # Find all plugins that want to register handlers + logger.debug('Loading autotools handlers') + handlers = [] + for plugin in plugins: + if hasattr(plugin, 'register_autotools_handlers'): + plugin.register_autotools_handlers(handlers) + + values = {} + inherits = [] + + # Hardcoded map, we also use a dynamic one based on what's in the sysroot + progmap = {'flex': 'flex-native', + 'bison': 'bison-native', + 'm4': 'm4-native', + 'tar': 'tar-native', + 'ar': 'binutils-native', + 'ranlib': 'binutils-native', + 'ld': 'binutils-native', + 'strip': 'binutils-native', + 'libtool': '', + 'autoconf': '', + 'autoheader': '', + 'automake': '', + 'uname': '', + 'rm': '', + 'cp': '', + 'mv': '', + 'find': '', + 'awk': '', + 'sed': '', + } + progclassmap = {'gconftool-2': 'gconf', + 'pkg-config': 'pkgconfig', + 'python': 'pythonnative', + 'python3': 'python3native', + 'perl': 'perlnative', + 'makeinfo': 'texinfo', + } + + pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') + pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*') + lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*') + libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*') + progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') + dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?') + ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*') + am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*') + define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)') + + defines = {} + def subst_defines(value): + newvalue = value + for define, defval in defines.iteritems(): + newvalue = newvalue.replace(define, defval) + if newvalue != value: + return subst_defines(newvalue) + return value + + def process_value(value): + value = value.replace('[', '').replace(']', '') + if value.startswith('m4_esyscmd(') or value.startswith('m4_esyscmd_s('): + cmd = subst_defines(value[value.index('(')+1:-1]) + try: + if '|' in cmd: + cmd = 'set -o pipefail; ' + cmd + stdout, _ = bb.process.run(cmd, cwd=srctree, shell=True) + ret = stdout.rstrip() + except bb.process.ExecutionError as e: + ret = '' + elif value.startswith('m4_'): + return None + ret = subst_defines(value) + if ret: + ret = ret.strip('"\'') + return ret + + # Since a configure.ac file is essentially a program, this is only ever going to be + # a hack unfortunately; but it ought to be enough of an approximation + if acfile: + srcfiles = [acfile] + else: + srcfiles = RecipeHandler.checkfiles(srctree, ['acinclude.m4', 'configure.ac', 'configure.in']) + + pcdeps = [] + libdeps = [] + deps = [] + unmapped = [] + + RecipeHandler.load_binmap(tinfoil.config_data) + + def process_macro(keyword, value): + for handler in handlers: + if handler.process_macro(srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values): + return + if keyword == 'PKG_CHECK_MODULES': + res = pkg_re.search(value) + if res: + res = dep_re.findall(res.group(1)) + if res: + pcdeps.extend([x[0] for x in res]) + inherits.append('pkgconfig') + elif keyword == 'PKG_CHECK_EXISTS': + res = pkgce_re.search(value) + if res: + res = dep_re.findall(res.group(1)) + if res: + pcdeps.extend([x[0] for x in res]) + inherits.append('pkgconfig') + elif keyword in ('AM_GNU_GETTEXT', 'AM_GLIB_GNU_GETTEXT', 'GETTEXT_PACKAGE'): + inherits.append('gettext') + elif keyword in ('AC_PROG_INTLTOOL', 'IT_PROG_INTLTOOL'): + deps.append('intltool-native') + elif keyword == 'AM_PATH_GLIB_2_0': + deps.append('glib-2.0') + elif keyword in ('AC_CHECK_PROG', 'AC_PATH_PROG', 'AX_WITH_PROG'): + res = progs_re.search(value) + if res: + for prog in shlex.split(res.group(1)): + prog = prog.split()[0] + for handler in handlers: + if handler.process_prog(srctree, keyword, value, prog, deps, outlines, inherits, values): + return + progclass = progclassmap.get(prog, None) + if progclass: + inherits.append(progclass) + else: + progdep = RecipeHandler.recipebinmap.get(prog, None) + if not progdep: + progdep = progmap.get(prog, None) + if progdep: + deps.append(progdep) + elif progdep is None: + if not prog.startswith('$'): + unmapped.append(prog) + elif keyword == 'AC_CHECK_LIB': + res = lib_re.search(value) + if res: + lib = res.group(1) + if not lib.startswith('$'): + libdeps.append(lib) + elif keyword == 'AX_CHECK_LIBRARY': + res = libx_re.search(value) + if res: + lib = res.group(2) + if not lib.startswith('$'): + header = res.group(1) + libdeps.append((lib, header)) + elif keyword == 'AC_PATH_X': + deps.append('libx11') + elif keyword in ('AX_BOOST', 'BOOST_REQUIRE'): + deps.append('boost') + elif keyword in ('AC_PROG_LEX', 'AM_PROG_LEX', 'AX_PROG_FLEX'): + deps.append('flex-native') + elif keyword in ('AC_PROG_YACC', 'AX_PROG_BISON'): + deps.append('bison-native') + elif keyword == 'AX_CHECK_ZLIB': + deps.append('zlib') + elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'): + deps.append('openssl') + elif keyword == 'AX_LIB_CURL': + deps.append('curl') + elif keyword == 'AX_LIB_BEECRYPT': + deps.append('beecrypt') + elif keyword == 'AX_LIB_EXPAT': + deps.append('expat') + elif keyword == 'AX_LIB_GCRYPT': + deps.append('libgcrypt') + elif keyword == 'AX_LIB_NETTLE': + deps.append('nettle') + elif keyword == 'AX_LIB_READLINE': + deps.append('readline') + elif keyword == 'AX_LIB_SQLITE3': + deps.append('sqlite3') + elif keyword == 'AX_LIB_TAGLIB': + deps.append('taglib') + elif keyword == 'AX_PKG_SWIG': + deps.append('swig') + elif keyword == 'AX_PROG_XSLTPROC': + deps.append('libxslt-native') + elif keyword == 'AX_WITH_CURSES': + deps.append('ncurses') + elif keyword == 'AX_PATH_BDB': + deps.append('db') + elif keyword == 'AX_PATH_LIB_PCRE': + deps.append('libpcre') + elif keyword == 'AC_INIT': + if extravalues is not None: + res = ac_init_re.match(value) + if res: + extravalues['PN'] = process_value(res.group(1)) + pv = process_value(res.group(2)) + if validate_pv(pv): + extravalues['PV'] = pv + elif keyword == 'AM_INIT_AUTOMAKE': + if extravalues is not None: + if 'PN' not in extravalues: + res = am_init_re.match(value) + if res: + if res.group(1) != 'AC_PACKAGE_NAME': + extravalues['PN'] = process_value(res.group(1)) + pv = process_value(res.group(2)) + if validate_pv(pv): + extravalues['PV'] = pv + elif keyword == 'define(': + res = define_re.match(value) + if res: + key = res.group(2).strip('[]') + value = process_value(res.group(3)) + if value is not None: + defines[key] = value + + keywords = ['PKG_CHECK_MODULES', + 'PKG_CHECK_EXISTS', + 'AM_GNU_GETTEXT', + 'AM_GLIB_GNU_GETTEXT', + 'GETTEXT_PACKAGE', + 'AC_PROG_INTLTOOL', + 'IT_PROG_INTLTOOL', + 'AM_PATH_GLIB_2_0', + 'AC_CHECK_PROG', + 'AC_PATH_PROG', + 'AX_WITH_PROG', + 'AC_CHECK_LIB', + 'AX_CHECK_LIBRARY', + 'AC_PATH_X', + 'AX_BOOST', + 'BOOST_REQUIRE', + 'AC_PROG_LEX', + 'AM_PROG_LEX', + 'AX_PROG_FLEX', + 'AC_PROG_YACC', + 'AX_PROG_BISON', + 'AX_CHECK_ZLIB', + 'AX_CHECK_OPENSSL', + 'AX_LIB_CRYPTO', + 'AX_LIB_CURL', + 'AX_LIB_BEECRYPT', + 'AX_LIB_EXPAT', + 'AX_LIB_GCRYPT', + 'AX_LIB_NETTLE', + 'AX_LIB_READLINE' + 'AX_LIB_SQLITE3', + 'AX_LIB_TAGLIB', + 'AX_PKG_SWIG', + 'AX_PROG_XSLTPROC', + 'AX_WITH_CURSES', + 'AX_PATH_BDB', + 'AX_PATH_LIB_PCRE', + 'AC_INIT', + 'AM_INIT_AUTOMAKE', + 'define(', + ] + + for handler in handlers: + handler.extend_keywords(keywords) + + for srcfile in srcfiles: + nesting = 0 + in_keyword = '' + partial = '' + with open(srcfile, 'r') as f: + for line in f: + if in_keyword: + partial += ' ' + line.strip() + if partial.endswith('\\'): + partial = partial[:-1] + nesting = nesting + line.count('(') - line.count(')') + if nesting == 0: + process_macro(in_keyword, partial) + partial = '' + in_keyword = '' + else: + for keyword in keywords: + if keyword in line: + nesting = line.count('(') - line.count(')') + if nesting > 0: + partial = line.strip() + if partial.endswith('\\'): + partial = partial[:-1] + in_keyword = keyword + else: + process_macro(keyword, line.strip()) + break + + if in_keyword: + process_macro(in_keyword, partial) + + if extravalues: + for k,v in extravalues.items(): + if v: + if v.startswith('$') or v.startswith('@') or v.startswith('%'): + del extravalues[k] + else: + extravalues[k] = v.strip('"\'').rstrip('()') + + if unmapped: + outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmapped)))) + + RecipeHandler.handle_depends(libdeps, pcdeps, deps, outlines, values, tinfoil.config_data) + + for handler in handlers: + handler.post_process(srctree, libdeps, pcdeps, deps, outlines, inherits, values) + + if inherits: + values['inherit'] = ' '.join(list(set(inherits))) + + return values + + +class AutotoolsExtensionHandler(object): + '''Base class for Autotools extension handlers''' + def process_macro(self, srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values): + ''' + Handle a macro parsed out of an autotools file. Note that if you want this to be called + for any macro other than the ones AutotoolsRecipeHandler already looks for, you'll need + to add it to the keywords list in extend_keywords(). + Return True if you've completely handled the passed in macro, otherwise return False. + ''' + return False + + def extend_keywords(self, keywords): + '''Adds keywords to be recognised by the parser (so that you get a call to process_macro)''' + return + + def process_prog(self, srctree, keyword, value, prog, deps, outlines, inherits, values): + ''' + Handle an AC_PATH_PROG, AC_CHECK_PROG etc. line + Return True if you've completely handled the passed in macro, otherwise return False. + ''' + return False + + def post_process(self, srctree, fn, pkg, deps, outlines, inherits, values): + ''' + Apply any desired post-processing on the output + ''' + return + + +class MakefileRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'buildsystem' in handled: + return False + + makefile = RecipeHandler.checkfiles(srctree, ['Makefile']) + if makefile: + lines_after.append('# NOTE: this is a Makefile-only piece of software, so we cannot generate much of the') + lines_after.append('# recipe automatically - you will need to examine the Makefile yourself and ensure') + lines_after.append('# that the appropriate arguments are passed in.') + lines_after.append('') + + scanfile = os.path.join(srctree, 'configure.scan') + skipscan = False + try: + stdout, stderr = bb.process.run('autoscan', cwd=srctree, shell=True) + except bb.process.ExecutionError as e: + skipscan = True + if scanfile and os.path.exists(scanfile): + values = AutotoolsRecipeHandler.extract_autotools_deps(lines_before, srctree, acfile=scanfile) + classes.extend(values.pop('inherit', '').split()) + for var, value in values.iteritems(): + if var == 'DEPENDS': + lines_before.append('# NOTE: some of these dependencies may be optional, check the Makefile and/or upstream documentation') + lines_before.append('%s = "%s"' % (var, value)) + lines_before.append('') + for f in ['configure.scan', 'autoscan.log']: + fp = os.path.join(srctree, f) + if os.path.exists(fp): + os.remove(fp) + + self.genfunction(lines_after, 'do_configure', ['# Specify any needed configure commands here']) + + func = [] + func.append('# You will almost certainly need to add additional arguments here') + func.append('oe_runmake') + self.genfunction(lines_after, 'do_compile', func) + + installtarget = True + try: + stdout, stderr = bb.process.run('make -n install', cwd=srctree, shell=True) + except bb.process.ExecutionError as e: + if e.exitcode != 1: + installtarget = False + func = [] + if installtarget: + func.append('# This is a guess; additional arguments may be required') + makeargs = '' + with open(makefile[0], 'r') as f: + for i in range(1, 100): + if 'DESTDIR' in f.readline(): + makeargs += " 'DESTDIR=${D}'" + break + func.append('oe_runmake install%s' % makeargs) + else: + func.append('# NOTE: unable to determine what to put here - there is a Makefile but no') + func.append('# target named "install", so you will need to define this yourself') + self.genfunction(lines_after, 'do_install', func) + + handled.append('buildsystem') + else: + lines_after.append('# NOTE: no Makefile found, unable to determine what needs to be done') + lines_after.append('') + self.genfunction(lines_after, 'do_configure', ['# Specify any needed configure commands here']) + self.genfunction(lines_after, 'do_compile', ['# Specify compilation commands here']) + self.genfunction(lines_after, 'do_install', ['# Specify install commands here']) + + +class VersionFileRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'PV' not in extravalues: + # Look for a VERSION or version file containing a single line consisting + # only of a version number + filelist = RecipeHandler.checkfiles(srctree, ['VERSION', 'version']) + version = None + for fileitem in filelist: + linecount = 0 + with open(fileitem, 'r') as f: + for line in f: + line = line.rstrip().strip('"\'') + linecount += 1 + if line: + if linecount > 1: + version = None + break + else: + if validate_pv(line): + version = line + if version: + extravalues['PV'] = version + break + + +class SpecFileRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'PV' in extravalues and 'PN' in extravalues: + return + filelist = RecipeHandler.checkfiles(srctree, ['*.spec'], recursive=True) + pn = None + pv = None + for fileitem in filelist: + linecount = 0 + with open(fileitem, 'r') as f: + for line in f: + if line.startswith('Name:') and not pn: + pn = line.split(':')[1].strip() + if line.startswith('Version:') and not pv: + pv = line.split(':')[1].strip() + if pv or pn: + if pv and not 'PV' in extravalues and validate_pv(pv): + extravalues['PV'] = pv + if pn and not 'PN' in extravalues: + extravalues['PN'] = pn + break + +def register_recipe_handlers(handlers): + # Set priorities with some gaps so that other plugins can insert + # their own handlers (so avoid changing these numbers) + handlers.append((CmakeRecipeHandler(), 50)) + handlers.append((AutotoolsRecipeHandler(), 40)) + handlers.append((SconsRecipeHandler(), 30)) + handlers.append((QmakeRecipeHandler(), 20)) + handlers.append((MakefileRecipeHandler(), 10)) + handlers.append((VersionFileRecipeHandler(), -1)) + handlers.append((SpecFileRecipeHandler(), -1)) diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py new file mode 100644 index 0000000..c382330 --- /dev/null +++ b/scripts/lib/recipetool/create_buildsys_python.py @@ -0,0 +1,719 @@ +# Recipe creation tool - create build system handler for python +# +# Copyright (C) 2015 Mentor Graphics Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import ast +import codecs +import collections +import distutils.command.build_py +import email +import imp +import glob +import itertools +import logging +import os +import re +import sys +import subprocess +from recipetool.create import RecipeHandler + +logger = logging.getLogger('recipetool') + +tinfoil = None + + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +class PythonRecipeHandler(RecipeHandler): + base_pkgdeps = ['python-core'] + excluded_pkgdeps = ['python-dbg'] + # os.path is provided by python-core + assume_provided = ['builtins', 'os.path'] + # Assumes that the host python builtin_module_names is sane for target too + assume_provided = assume_provided + list(sys.builtin_module_names) + + bbvar_map = { + 'Name': 'PN', + 'Version': 'PV', + 'Home-page': 'HOMEPAGE', + 'Summary': 'SUMMARY', + 'Description': 'DESCRIPTION', + 'License': 'LICENSE', + 'Requires': 'RDEPENDS_${PN}', + 'Provides': 'RPROVIDES_${PN}', + 'Obsoletes': 'RREPLACES_${PN}', + } + # PN/PV are already set by recipetool core & desc can be extremely long + excluded_fields = [ + 'Name', + 'Version', + 'Description', + ] + setup_parse_map = { + 'Url': 'Home-page', + 'Classifiers': 'Classifier', + 'Description': 'Summary', + } + setuparg_map = { + 'Home-page': 'url', + 'Classifier': 'classifiers', + 'Summary': 'description', + 'Description': 'long-description', + } + # Values which are lists, used by the setup.py argument based metadata + # extraction method, to determine how to process the setup.py output. + setuparg_list_fields = [ + 'Classifier', + 'Requires', + 'Provides', + 'Obsoletes', + 'Platform', + 'Supported-Platform', + ] + setuparg_multi_line_values = ['Description'] + replacements = [ + ('License', r' ', '-'), + ('License', r'-License$', ''), + ('License', r'^UNKNOWN$', ''), + + # Remove currently unhandled version numbers from these variables + ('Requires', r' *\([^)]*\)', ''), + ('Provides', r' *\([^)]*\)', ''), + ('Obsoletes', r' *\([^)]*\)', ''), + ('Install-requires', r'^([^><= ]+).*', r'\1'), + ('Extras-require', r'^([^><= ]+).*', r'\1'), + ('Tests-require', r'^([^><= ]+).*', r'\1'), + + # Remove unhandled dependency on particular features (e.g. foo[PDF]) + ('Install-requires', r'\[[^\]]+\]$', ''), + ] + + classifier_license_map = { + 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL', + 'License :: OSI Approved :: Apache Software License': 'Apache', + 'License :: OSI Approved :: Apple Public Source License': 'APSL', + 'License :: OSI Approved :: Artistic License': 'Artistic', + 'License :: OSI Approved :: Attribution Assurance License': 'AAL', + 'License :: OSI Approved :: BSD License': 'BSD', + 'License :: OSI Approved :: Common Public License': 'CPL', + 'License :: OSI Approved :: Eiffel Forum License': 'EFL', + 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0', + 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1', + 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0+', + 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0', + 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL', + 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL', + 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0', + 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0+', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0', + 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0+', + 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0', + 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0+', + 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0', + 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0+', + 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL', + 'License :: OSI Approved :: IBM Public License': 'IPL', + 'License :: OSI Approved :: ISC License (ISCL)': 'ISC', + 'License :: OSI Approved :: Intel Open Source License': 'Intel', + 'License :: OSI Approved :: Jabber Open Source License': 'Jabber', + 'License :: OSI Approved :: MIT License': 'MIT', + 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL', + 'License :: OSI Approved :: Motosoto License': 'Motosoto', + 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0', + 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1', + 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0', + 'License :: OSI Approved :: Nethack General Public License': 'NGPL', + 'License :: OSI Approved :: Nokia Open Source License': 'Nokia', + 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL', + 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python', + 'License :: OSI Approved :: Python Software Foundation License': 'PSF', + 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL', + 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL', + 'License :: OSI Approved :: Sleepycat License': 'Sleepycat', + 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': '-- Sun Industry Standards Source License (SISSL)', + 'License :: OSI Approved :: Sun Public License': 'SPL', + 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA', + 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0', + 'License :: OSI Approved :: W3C License': 'W3C', + 'License :: OSI Approved :: X.Net License': 'Xnet', + 'License :: OSI Approved :: Zope Public License': 'ZPL', + 'License :: OSI Approved :: zlib/libpng License': 'Zlib', + } + + def __init__(self): + pass + + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'buildsystem' in handled: + return False + + if not RecipeHandler.checkfiles(srctree, ['setup.py']): + return + + # setup.py is always parsed to get at certain required information, such as + # distutils vs setuptools + # + # If egg info is available, we use it for both its PKG-INFO metadata + # and for its requires.txt for install_requires. + # If PKG-INFO is available but no egg info is, we use that for metadata in preference to + # the parsed setup.py, but use the install_requires info from the + # parsed setup.py. + + setupscript = os.path.join(srctree, 'setup.py') + try: + setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript) + except Exception: + logger.exception("Failed to parse setup.py") + setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], [] + + egginfo = glob.glob(os.path.join(srctree, '*.egg-info')) + if egginfo: + info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO')) + requires_txt = os.path.join(egginfo[0], 'requires.txt') + if os.path.exists(requires_txt): + with codecs.open(requires_txt) as f: + inst_req = [] + extras_req = collections.defaultdict(list) + current_feature = None + for line in f.readlines(): + line = line.rstrip() + if not line: + continue + + if line.startswith('['): + current_feature = line[1:-1] + elif current_feature: + extras_req[current_feature].append(line) + else: + inst_req.append(line) + info['Install-requires'] = inst_req + info['Extras-require'] = extras_req + elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']): + info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO')) + + if setup_info: + if 'Install-requires' in setup_info: + info['Install-requires'] = setup_info['Install-requires'] + if 'Extras-require' in setup_info: + info['Extras-require'] = setup_info['Extras-require'] + else: + if setup_info: + info = setup_info + else: + info = self.get_setup_args_info(setupscript) + + self.apply_info_replacements(info) + + if uses_setuptools: + classes.append('setuptools') + else: + classes.append('distutils') + + if 'Classifier' in info: + licenses = [] + for classifier in info['Classifier']: + if classifier in self.classifier_license_map: + license = self.classifier_license_map[classifier] + licenses.append(license) + + if licenses: + info['License'] = ' & '.join(licenses) + + + # Map PKG-INFO & setup.py fields to bitbake variables + bbinfo = {} + for field, values in info.iteritems(): + if field in self.excluded_fields: + continue + + if field not in self.bbvar_map: + continue + + if isinstance(values, basestring): + value = values + else: + value = ' '.join(str(v) for v in values if v) + + bbvar = self.bbvar_map[field] + if bbvar not in bbinfo and value: + bbinfo[bbvar] = value + + comment_lic_line = None + for pos, line in enumerate(list(lines_before)): + if line.startswith('#') and 'LICENSE' in line: + comment_lic_line = pos + elif line.startswith('LICENSE =') and 'LICENSE' in bbinfo: + if line in ('LICENSE = "Unknown"', 'LICENSE = "CLOSED"'): + lines_before[pos] = 'LICENSE = "{}"'.format(bbinfo['LICENSE']) + if line == 'LICENSE = "CLOSED"' and comment_lic_line: + lines_before[comment_lic_line:pos] = [ + '# WARNING: the following LICENSE value is a best guess - it is your', + '# responsibility to verify that the value is complete and correct.' + ] + del bbinfo['LICENSE'] + + src_uri_line = None + for pos, line in enumerate(lines_before): + if line.startswith('SRC_URI ='): + src_uri_line = pos + + if bbinfo: + mdinfo = [''] + for k in sorted(bbinfo): + v = bbinfo[k] + mdinfo.append('{} = "{}"'.format(k, v)) + if src_uri_line: + lines_before[src_uri_line-1:src_uri_line-1] = mdinfo + else: + lines_before.extend(mdinfo) + + mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) + + extras_req = set() + if 'Extras-require' in info: + extras_req = info['Extras-require'] + if extras_req: + lines_after.append('# The following configs & dependencies are from setuptools extras_require.') + lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') + lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') + lines_after.append('#') + lines_after.append('# Uncomment this line to enable all the optional features.') + lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys()))) + for feature, feature_reqs in extras_req.iteritems(): + unmapped_deps.difference_update(feature_reqs) + + feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) + lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) + + inst_reqs = set() + if 'Install-requires' in info: + if extras_req: + lines_after.append('') + inst_reqs = info['Install-requires'] + if inst_reqs: + unmapped_deps.difference_update(inst_reqs) + + inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) + lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') + lines_after.append('# upstream names may not correspond exactly to bitbake package names.') + lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) + + if mapped_deps: + name = info.get('Name') + if name and name[0] in mapped_deps: + # Attempt to avoid self-reference + mapped_deps.remove(name[0]) + mapped_deps -= set(self.excluded_pkgdeps) + if inst_reqs or extras_req: + lines_after.append('') + lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') + lines_after.append('# python sources, and might not be 100% accurate.') + lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) + + unmapped_deps -= set(extensions) + unmapped_deps -= set(self.assume_provided) + if unmapped_deps: + if mapped_deps: + lines_after.append('') + lines_after.append('# WARNING: We were unable to map the following python package/module') + lines_after.append('# dependencies to the bitbake packages which include them:') + lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) + + handled.append('buildsystem') + + def get_pkginfo(self, pkginfo_fn): + msg = email.message_from_file(open(pkginfo_fn, 'r')) + msginfo = {} + for field in msg.keys(): + values = msg.get_all(field) + if len(values) == 1: + msginfo[field] = values[0] + else: + msginfo[field] = values + return msginfo + + def parse_setup_py(self, setupscript='./setup.py'): + with codecs.open(setupscript) as f: + info, imported_modules, non_literals, extensions = gather_setup_info(f) + + def _map(key): + key = key.replace('_', '-') + key = key[0].upper() + key[1:] + if key in self.setup_parse_map: + key = self.setup_parse_map[key] + return key + + # Naive mapping of setup() arguments to PKG-INFO field names + for d in [info, non_literals]: + for key, value in d.items(): + new_key = _map(key) + if new_key != key: + del d[key] + d[new_key] = value + + return info, 'setuptools' in imported_modules, non_literals, extensions + + def get_setup_args_info(self, setupscript='./setup.py'): + cmd = ['python', setupscript] + info = {} + keys = set(self.bbvar_map.keys()) + keys |= set(self.setuparg_list_fields) + keys |= set(self.setuparg_multi_line_values) + grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values)) + for index, keys in grouped_keys: + if index == (True, False): + # Splitlines output for each arg as a list value + for key in keys: + arg = self.setuparg_map.get(key, key.lower()) + try: + arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) + except (OSError, subprocess.CalledProcessError): + pass + else: + info[key] = [l.rstrip() for l in arg_info.splitlines()] + elif index == (False, True): + # Entire output for each arg + for key in keys: + arg = self.setuparg_map.get(key, key.lower()) + try: + arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) + except (OSError, subprocess.CalledProcessError): + pass + else: + info[key] = arg_info + else: + info.update(self.get_setup_byline(list(keys), setupscript)) + return info + + def get_setup_byline(self, fields, setupscript='./setup.py'): + info = {} + + cmd = ['python', setupscript] + cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields) + try: + info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines() + except (OSError, subprocess.CalledProcessError): + pass + else: + if len(fields) != len(info_lines): + logger.error('Mismatch between setup.py output lines and number of fields') + sys.exit(1) + + for lineno, line in enumerate(info_lines): + line = line.rstrip() + info[fields[lineno]] = line + return info + + def apply_info_replacements(self, info): + for variable, search, replace in self.replacements: + if variable not in info: + continue + + def replace_value(search, replace, value): + if replace is None: + if re.search(search, value): + return None + else: + new_value = re.sub(search, replace, value) + if value != new_value: + return new_value + return value + + value = info[variable] + if isinstance(value, basestring): + new_value = replace_value(search, replace, value) + if new_value is None: + del info[variable] + elif new_value != value: + info[variable] = new_value + elif hasattr(value, 'iteritems'): + for dkey, dvalue in value.iteritems(): + new_list = [] + for pos, a_value in enumerate(dvalue): + new_value = replace_value(search, replace, a_value) + if new_value is not None and new_value != value: + new_list.append(new_value) + + if value != new_list: + value[dkey] = new_list + else: + new_list = [] + for pos, a_value in enumerate(value): + new_value = replace_value(search, replace, a_value) + if new_value is not None and new_value != value: + new_list.append(new_value) + + if value != new_list: + info[variable] = new_list + + def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): + if 'Package-dir' in setup_info: + package_dir = setup_info['Package-dir'] + else: + package_dir = {} + + class PackageDir(distutils.command.build_py.build_py): + def __init__(self, package_dir): + self.package_dir = package_dir + + pd = PackageDir(package_dir) + to_scan = [] + if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']): + if 'Py-modules' in setup_info: + for module in setup_info['Py-modules']: + try: + package, module = module.rsplit('.', 1) + except ValueError: + package, module = '.', module + module_path = os.path.join(pd.get_package_dir(package), module + '.py') + to_scan.append(module_path) + + if 'Packages' in setup_info: + for package in setup_info['Packages']: + to_scan.append(pd.get_package_dir(package)) + + if 'Scripts' in setup_info: + to_scan.extend(setup_info['Scripts']) + else: + logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.") + + if not to_scan: + to_scan = ['.'] + + logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan)) + + provided_packages = self.parse_pkgdata_for_python_packages() + scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan]) + mapped_deps, unmapped_deps = set(self.base_pkgdeps), set() + for dep in scanned_deps: + mapped = provided_packages.get(dep) + if mapped: + mapped_deps.add(mapped) + else: + unmapped_deps.add(dep) + return mapped_deps, unmapped_deps + + def scan_python_dependencies(self, paths): + deps = set() + try: + dep_output = self.run_command(['pythondeps', '-d'] + paths) + except (OSError, subprocess.CalledProcessError): + pass + else: + for line in dep_output.splitlines(): + line = line.rstrip() + dep, filename = line.split('\t', 1) + if filename.endswith('/setup.py'): + continue + deps.add(dep) + + try: + provides_output = self.run_command(['pythondeps', '-p'] + paths) + except (OSError, subprocess.CalledProcessError): + pass + else: + provides_lines = (l.rstrip() for l in provides_output.splitlines()) + provides = set(l for l in provides_lines if l and l != 'setup') + deps -= provides + + return deps + + def parse_pkgdata_for_python_packages(self): + suffixes = [t[0] for t in imp.get_suffixes()] + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) + + ldata = tinfoil.config_data.createCopy() + bb.parse.handle('classes/python-dir.bbclass', ldata, True) + python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True) + + dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') + python_dirs = [python_sitedir + os.sep, + os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, + os.path.dirname(python_sitedir) + os.sep] + packages = {} + for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): + files_info = None + with open(pkgdatafile, 'r') as f: + for line in f.readlines(): + field, value = line.split(': ', 1) + if field == 'FILES_INFO': + files_info = ast.literal_eval(value) + break + else: + continue + + for fn in files_info.iterkeys(): + for suffix in suffixes: + if fn.endswith(suffix): + break + else: + continue + + if fn.startswith(dynload_dir + os.sep): + base = os.path.basename(fn) + provided = base.split('.', 1)[0] + packages[provided] = os.path.basename(pkgdatafile) + continue + + for python_dir in python_dirs: + if fn.startswith(python_dir): + relpath = fn[len(python_dir):] + relstart, _, relremaining = relpath.partition(os.sep) + if relstart.endswith('.egg'): + relpath = relremaining + base, _ = os.path.splitext(relpath) + + if '/.debug/' in base: + continue + if os.path.basename(base) == '__init__': + base = os.path.dirname(base) + base = base.replace(os.sep + os.sep, os.sep) + provided = base.replace(os.sep, '.') + packages[provided] = os.path.basename(pkgdatafile) + return packages + + @classmethod + def run_command(cls, cmd, **popenargs): + if 'stderr' not in popenargs: + popenargs['stderr'] = subprocess.STDOUT + try: + return subprocess.check_output(cmd, **popenargs) + except OSError as exc: + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) + raise + except subprocess.CalledProcessError as exc: + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) + raise + + +def gather_setup_info(fileobj): + parsed = ast.parse(fileobj.read(), fileobj.name) + visitor = SetupScriptVisitor() + visitor.visit(parsed) + + non_literals, extensions = {}, [] + for key, value in visitor.keywords.items(): + if key == 'ext_modules': + if isinstance(value, list): + for ext in value: + if (isinstance(ext, ast.Call) and + isinstance(ext.func, ast.Name) and + ext.func.id == 'Extension' and + not has_non_literals(ext.args)): + extensions.append(ext.args[0]) + elif has_non_literals(value): + non_literals[key] = value + del visitor.keywords[key] + + return visitor.keywords, visitor.imported_modules, non_literals, extensions + + +class SetupScriptVisitor(ast.NodeVisitor): + def __init__(self): + ast.NodeVisitor.__init__(self) + self.keywords = {} + self.non_literals = [] + self.imported_modules = set() + + def visit_Expr(self, node): + if isinstance(node.value, ast.Call) and \ + isinstance(node.value.func, ast.Name) and \ + node.value.func.id == 'setup': + self.visit_setup(node.value) + + def visit_setup(self, node): + call = LiteralAstTransform().visit(node) + self.keywords = call.keywords + for k, v in self.keywords.iteritems(): + if has_non_literals(v): + self.non_literals.append(k) + + def visit_Import(self, node): + for alias in node.names: + self.imported_modules.add(alias.name) + + def visit_ImportFrom(self, node): + self.imported_modules.add(node.module) + + +class LiteralAstTransform(ast.NodeTransformer): + """Simplify the ast through evaluation of literals.""" + excluded_fields = ['ctx'] + + def visit(self, node): + if not isinstance(node, ast.AST): + return node + else: + return ast.NodeTransformer.visit(self, node) + + def generic_visit(self, node): + try: + return ast.literal_eval(node) + except ValueError: + for field, value in ast.iter_fields(node): + if field in self.excluded_fields: + delattr(node, field) + if value is None: + continue + + if isinstance(value, list): + if field in ('keywords', 'kwargs'): + new_value = dict((kw.arg, self.visit(kw.value)) for kw in value) + else: + new_value = [self.visit(i) for i in value] + else: + new_value = self.visit(value) + setattr(node, field, new_value) + return node + + def visit_Name(self, node): + if hasattr('__builtins__', node.id): + return getattr(__builtins__, node.id) + else: + return self.generic_visit(node) + + def visit_Tuple(self, node): + return tuple(self.visit(v) for v in node.elts) + + def visit_List(self, node): + return [self.visit(v) for v in node.elts] + + def visit_Set(self, node): + return set(self.visit(v) for v in node.elts) + + def visit_Dict(self, node): + keys = (self.visit(k) for k in node.keys) + values = (self.visit(v) for v in node.values) + return dict(zip(keys, values)) + + +def has_non_literals(value): + if isinstance(value, ast.AST): + return True + elif isinstance(value, basestring): + return False + elif hasattr(value, 'itervalues'): + return any(has_non_literals(v) for v in value.itervalues()) + elif hasattr(value, '__iter__'): + return any(has_non_literals(v) for v in value) + + +def register_recipe_handlers(handlers): + # We need to make sure this is ahead of the makefile fallback handler + handlers.append((PythonRecipeHandler(), 70)) diff --git a/scripts/lib/recipetool/create_kernel.py b/scripts/lib/recipetool/create_kernel.py new file mode 100644 index 0000000..c6e86bd --- /dev/null +++ b/scripts/lib/recipetool/create_kernel.py @@ -0,0 +1,99 @@ +# Recipe creation tool - kernel support plugin +# +# Copyright (C) 2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import re +import logging +from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv + +logger = logging.getLogger('recipetool') + +tinfoil = None + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +class KernelRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + import bb.process + if 'buildsystem' in handled: + return False + + for tell in ['arch', 'firmware', 'Kbuild', 'Kconfig']: + if not os.path.exists(os.path.join(srctree, tell)): + return False + + handled.append('buildsystem') + del lines_after[:] + del classes[:] + template = os.path.join(tinfoil.config_data.getVar('COREBASE', True), 'meta-skeleton', 'recipes-kernel', 'linux', 'linux-yocto-custom.bb') + def handle_var(varname, origvalue, op, newlines): + if varname in ['SRCREV', 'SRCREV_machine']: + while newlines[-1].startswith('#'): + del newlines[-1] + try: + stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree, shell=True) + except bb.process.ExecutionError as e: + stdout = None + if stdout: + return stdout.strip(), op, 0, True + elif varname == 'LINUX_VERSION': + makefile = os.path.join(srctree, 'Makefile') + if os.path.exists(makefile): + kversion = -1 + kpatchlevel = -1 + ksublevel = -1 + kextraversion = '' + with open(makefile, 'r') as f: + for i, line in enumerate(f): + if i > 10: + break + if line.startswith('VERSION ='): + kversion = int(line.split('=')[1].strip()) + elif line.startswith('PATCHLEVEL ='): + kpatchlevel = int(line.split('=')[1].strip()) + elif line.startswith('SUBLEVEL ='): + ksublevel = int(line.split('=')[1].strip()) + elif line.startswith('EXTRAVERSION ='): + kextraversion = line.split('=')[1].strip() + version = '' + if kversion > -1 and kpatchlevel > -1: + version = '%d.%d' % (kversion, kpatchlevel) + if ksublevel > -1: + version += '.%d' % ksublevel + version += kextraversion + if version: + return version, op, 0, True + elif varname == 'SRC_URI': + while newlines[-1].startswith('#'): + del newlines[-1] + elif varname == 'COMPATIBLE_MACHINE': + while newlines[-1].startswith('#'): + del newlines[-1] + machine = tinfoil.config_data.getVar('MACHINE', True) + return machine, op, 0, True + return origvalue, op, 0, True + with open(template, 'r') as f: + varlist = ['SRCREV', 'SRCREV_machine', 'SRC_URI', 'LINUX_VERSION', 'COMPATIBLE_MACHINE'] + (_, newlines) = bb.utils.edit_metadata(f, varlist, handle_var) + lines_before[:] = [line.rstrip('\n') for line in newlines] + + return True + +def register_recipe_handlers(handlers): + handlers.append((KernelRecipeHandler(), 100)) diff --git a/scripts/lib/recipetool/create_kmod.py b/scripts/lib/recipetool/create_kmod.py new file mode 100644 index 0000000..fe39edb --- /dev/null +++ b/scripts/lib/recipetool/create_kmod.py @@ -0,0 +1,152 @@ +# Recipe creation tool - kernel module support plugin +# +# Copyright (C) 2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import re +import logging +from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv + +logger = logging.getLogger('recipetool') + +tinfoil = None + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +class KernelModuleRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + import bb.process + if 'buildsystem' in handled: + return False + + module_inc_re = re.compile(r'^#include\s+<linux/module.h>$') + makefiles = [] + is_module = False + + makefiles = [] + + files = RecipeHandler.checkfiles(srctree, ['*.c', '*.h'], recursive=True) + if files: + for cfile in files: + # Look in same dir or parent for Makefile + for makefile in [os.path.join(os.path.dirname(cfile), 'Makefile'), os.path.join(os.path.dirname(os.path.dirname(cfile)), 'Makefile')]: + if makefile in makefiles: + break + else: + if os.path.exists(makefile): + makefiles.append(makefile) + break + else: + continue + with open(cfile, 'r') as f: + for line in f: + if module_inc_re.match(line.strip()): + is_module = True + break + if is_module: + break + + if is_module: + classes.append('module') + handled.append('buildsystem') + # module.bbclass and the classes it inherits do most of the hard + # work, but we need to tweak it slightly depending on what the + # Makefile does (and there is a range of those) + # Check the makefile for the appropriate install target + install_lines = [] + compile_lines = [] + in_install = False + in_compile = False + install_target = None + with open(makefile, 'r') as f: + for line in f: + if line.startswith('install:'): + if not install_lines: + in_install = True + install_target = 'install' + elif line.startswith('modules_install:'): + install_lines = [] + in_install = True + install_target = 'modules_install' + elif line.startswith('modules:'): + compile_lines = [] + in_compile = True + elif line.startswith(('all:', 'default:')): + if not compile_lines: + in_compile = True + elif line: + if line[0] == '\t': + if in_install: + install_lines.append(line) + elif in_compile: + compile_lines.append(line) + elif ':' in line: + in_install = False + in_compile = False + + def check_target(lines, install): + kdirpath = '' + manual_install = False + for line in lines: + splitline = line.split() + if splitline[0] in ['make', 'gmake', '$(MAKE)']: + if '-C' in splitline: + idx = splitline.index('-C') + 1 + if idx < len(splitline): + kdirpath = splitline[idx] + break + elif install and splitline[0] == 'install': + if '.ko' in line: + manual_install = True + return kdirpath, manual_install + + kdirpath = None + manual_install = False + if install_lines: + kdirpath, manual_install = check_target(install_lines, install=True) + if compile_lines and not kdirpath: + kdirpath, _ = check_target(compile_lines, install=False) + + if manual_install or not install_lines: + lines_after.append('EXTRA_OEMAKE_append_task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"') + elif install_target and install_target != 'modules_install': + lines_after.append('MODULES_INSTALL_TARGET = "install"') + + warnmsg = None + kdirvar = None + if kdirpath: + res = re.match(r'\$\(([^$)]+)\)', kdirpath) + if res: + kdirvar = res.group(1) + if kdirvar != 'KERNEL_SRC': + lines_after.append('EXTRA_OEMAKE += "%s=${STAGING_KERNEL_DIR}"' % kdirvar) + elif kdirpath.startswith('/lib/'): + warnmsg = 'Kernel path in install makefile is hardcoded - you will need to patch the makefile' + if not kdirvar and not warnmsg: + warnmsg = 'Unable to find means of passing kernel path into install makefile - if kernel path is hardcoded you will need to patch the makefile' + if warnmsg: + warnmsg += '. Note that the variable KERNEL_SRC will be passed in as the kernel source path.' + logger.warn(warnmsg) + lines_after.append('# %s' % warnmsg) + + return True + + return False + +def register_recipe_handlers(handlers): + handlers.append((KernelModuleRecipeHandler(), 15)) diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py new file mode 100644 index 0000000..cc4fb42 --- /dev/null +++ b/scripts/lib/recipetool/create_npm.py @@ -0,0 +1,156 @@ +# Recipe creation tool - node.js NPM module support plugin +# +# Copyright (C) 2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import logging +import subprocess +import tempfile +import shutil +import json +from recipetool.create import RecipeHandler, split_pkg_licenses + +logger = logging.getLogger('recipetool') + + +tinfoil = None + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +class NpmRecipeHandler(RecipeHandler): + lockdownpath = None + + def _handle_license(self, data): + ''' + Handle the license value from an npm package.json file + ''' + license = None + if 'license' in data: + license = data['license'] + if isinstance(license, dict): + license = license.get('type', None) + return license + + def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before): + try: + runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True)) + bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) + except bb.process.ExecutionError as e: + logger.warn('npm shrinkwrap failed:\n%s' % e.stdout) + return + + tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json') + shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile) + extravalues.setdefault('extrafiles', {}) + extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile + lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"') + + def _lockdown(self, srctree, localfilesdir, extravalues, lines_before): + runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True)) + if not NpmRecipeHandler.lockdownpath: + NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown') + bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath, + cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) + relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js') + if not os.path.exists(relockbin): + logger.warn('Could not find relock.js within lockdown directory; skipping lockdown') + return + try: + bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) + except bb.process.ExecutionError as e: + logger.warn('lockdown-relock failed:\n%s' % e.stdout) + return + + tmpfile = os.path.join(localfilesdir, 'lockdown.json') + shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile) + extravalues.setdefault('extrafiles', {}) + extravalues['extrafiles']['lockdown.json'] = tmpfile + lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"') + + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + import bb.utils + import oe + from collections import OrderedDict + + if 'buildsystem' in handled: + return False + + def read_package_json(fn): + with open(fn, 'r') as f: + return json.loads(f.read()) + + files = RecipeHandler.checkfiles(srctree, ['package.json']) + if files: + data = read_package_json(files[0]) + if 'name' in data and 'version' in data: + extravalues['PN'] = data['name'] + extravalues['PV'] = data['version'] + classes.append('npm') + handled.append('buildsystem') + if 'description' in data: + lines_before.append('SUMMARY = "%s"' % data['description']) + if 'homepage' in data: + lines_before.append('HOMEPAGE = "%s"' % data['homepage']) + + # Shrinkwrap + localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm') + self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before) + + # Lockdown + self._lockdown(srctree, localfilesdir, extravalues, lines_before) + + # Split each npm module out to is own package + npmpackages = oe.package.npm_split_package_dirs(srctree) + for item in handled: + if isinstance(item, tuple): + if item[0] == 'license': + licvalues = item[1] + break + if licvalues: + # Augment the license list with information we have in the packages + licenses = {} + license = self._handle_license(data) + if license: + licenses['${PN}'] = license + for pkgname, pkgitem in npmpackages.iteritems(): + _, pdata = pkgitem + license = self._handle_license(pdata) + if license: + licenses[pkgname] = license + # Now write out the package-specific license values + # We need to strip out the json data dicts for this since split_pkg_licenses + # isn't expecting it + packages = OrderedDict((x,y[0]) for x,y in npmpackages.iteritems()) + packages['${PN}'] = '' + pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses) + all_licenses = list(set([item for pkglicense in pkglicenses.values() for item in pkglicense])) + # Go back and update the LICENSE value since we have a bit more + # information than when that was written out (and we know all apply + # vs. there being a choice, so we can join them with &) + for i, line in enumerate(lines_before): + if line.startswith('LICENSE = '): + lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses) + break + + return True + + return False + +def register_recipe_handlers(handlers): + handlers.append((NpmRecipeHandler(), 60)) diff --git a/scripts/lib/recipetool/newappend.py b/scripts/lib/recipetool/newappend.py new file mode 100644 index 0000000..4fbb40a --- /dev/null +++ b/scripts/lib/recipetool/newappend.py @@ -0,0 +1,112 @@ +# Recipe creation tool - newappend plugin +# +# This sub-command creates a bbappend for the specified target and prints the +# path to the bbappend. +# +# Example: recipetool newappend meta-mylayer busybox +# +# Copyright (C) 2015 Christopher Larson <kergoth@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import argparse +import errno +import logging +import os +import re +import subprocess +import sys +import scriptutils + + +logger = logging.getLogger('recipetool') +tinfoil = None + + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +def _provide_to_pn(cooker, provide): + """Get the name of the preferred recipe for the specified provide.""" + import bb.providers + filenames = cooker.recipecache.providers[provide] + eligible, foundUnique = bb.providers.filterProviders(filenames, provide, cooker.expanded_data, cooker.recipecache) + filename = eligible[0] + pn = cooker.recipecache.pkg_fn[filename] + return pn + + +def _get_recipe_file(cooker, pn): + import oe.recipeutils + recipefile = oe.recipeutils.pn_to_recipe(cooker, pn) + if not recipefile: + skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn) + if skipreasons: + logger.error('\n'.join(skipreasons)) + else: + logger.error("Unable to find any recipe file matching %s" % pn) + return recipefile + + +def layer(layerpath): + if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')): + raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath)) + return layerpath + + +def newappend(args): + import oe.recipeutils + + pn = _provide_to_pn(tinfoil.cooker, args.target) + recipe_path = _get_recipe_file(tinfoil.cooker, pn) + + rd = tinfoil.config_data.createCopy() + rd.setVar('FILE', recipe_path) + append_path, path_ok = oe.recipeutils.get_bbappend_path(rd, args.destlayer, args.wildcard_version) + if not append_path: + logger.error('Unable to determine layer directory containing %s', recipe_path) + return 1 + + if not path_ok: + logger.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.', os.path.join(args.destlayer, 'conf', 'layer.conf'), os.path.dirname(append_path)) + + layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS', True).split()] + if not os.path.abspath(args.destlayer) in layerdirs: + logger.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active') + + if not os.path.exists(append_path): + bb.utils.mkdirhier(os.path.dirname(append_path)) + + try: + open(append_path, 'a').close() + except (OSError, IOError) as exc: + logger.critical(str(exc)) + return 1 + + if args.edit: + return scriptutils.run_editor([append_path, recipe_path]) + else: + print(append_path) + + +def register_commands(subparsers): + parser = subparsers.add_parser('newappend', + help='Create a bbappend for the specified target in the specified layer') + parser.add_argument('-e', '--edit', help='Edit the new append. This obeys $VISUAL if set, otherwise $EDITOR, otherwise vi.', action='store_true') + parser.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true') + parser.add_argument('destlayer', help='Base directory of the destination layer to write the bbappend to', type=layer) + parser.add_argument('target', help='Target recipe/provide to append') + parser.set_defaults(func=newappend, parserecipes=True) diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py new file mode 100644 index 0000000..657d2b6 --- /dev/null +++ b/scripts/lib/recipetool/setvar.py @@ -0,0 +1,75 @@ +# Recipe creation tool - set variable plugin +# +# Copyright (C) 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import os +import argparse +import glob +import fnmatch +import re +import logging +import scriptutils + +logger = logging.getLogger('recipetool') + +tinfoil = None +plugins = None + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + +def setvar(args): + import oe.recipeutils + + if args.delete: + if args.value: + logger.error('-D/--delete and specifying a value are mutually exclusive') + return 1 + value = None + else: + if args.value is None: + logger.error('You must specify a value if not using -D/--delete') + return 1 + value = args.value + varvalues = {args.varname: value} + + if args.recipe_only: + patches = [oe.recipeutils.patch_recipe_file(args.recipefile, varvalues, patch=args.patch)] + else: + rd = oe.recipeutils.parse_recipe(args.recipefile, None, tinfoil.config_data) + if not rd: + return 1 + patches = oe.recipeutils.patch_recipe(rd, args.recipefile, varvalues, patch=args.patch) + if args.patch: + for patch in patches: + for line in patch: + sys.stdout.write(line) + return 0 + + +def register_commands(subparsers): + parser_setvar = subparsers.add_parser('setvar', + help='Set a variable within a recipe', + description='Adds/updates the value a variable is set to in a recipe') + parser_setvar.add_argument('recipefile', help='Recipe file to update') + parser_setvar.add_argument('varname', help='Variable name to set') + parser_setvar.add_argument('value', nargs='?', help='New value to set the variable to') + parser_setvar.add_argument('--recipe-only', '-r', help='Do not set variable in any include file if present', action='store_true') + parser_setvar.add_argument('--patch', '-p', help='Create a patch to make the change instead of modifying the recipe', action='store_true') + parser_setvar.add_argument('--delete', '-D', help='Delete the specified value instead of setting it', action='store_true') + parser_setvar.set_defaults(func=setvar) diff --git a/scripts/lib/scriptpath.py b/scripts/lib/scriptpath.py new file mode 100644 index 0000000..d00317e --- /dev/null +++ b/scripts/lib/scriptpath.py @@ -0,0 +1,42 @@ +# Path utility functions for OE python scripts +# +# Copyright (C) 2012-2014 Intel Corporation +# Copyright (C) 2011 Mentor Graphics Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import os +import os.path + +def add_oe_lib_path(): + basepath = os.path.abspath(os.path.dirname(__file__) + '/../..') + newpath = basepath + '/meta/lib' + sys.path.insert(0, newpath) + +def add_bitbake_lib_path(): + basepath = os.path.abspath(os.path.dirname(__file__) + '/../..') + bitbakepath = None + if os.path.exists(basepath + '/bitbake/lib/bb'): + bitbakepath = basepath + '/bitbake' + else: + # look for bitbake/bin dir in PATH + for pth in os.environ['PATH'].split(':'): + if os.path.exists(os.path.join(pth, '../lib/bb')): + bitbakepath = os.path.abspath(os.path.join(pth, '..')) + break + + if bitbakepath: + sys.path.insert(0, bitbakepath + '/lib') + return bitbakepath diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py new file mode 100644 index 0000000..aef19d3 --- /dev/null +++ b/scripts/lib/scriptutils.py @@ -0,0 +1,118 @@ +# Script utility functions +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import os +import logging +import glob +import argparse +import subprocess + +def logger_create(name): + logger = logging.getLogger(name) + loggerhandler = logging.StreamHandler() + loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + logger.addHandler(loggerhandler) + logger.setLevel(logging.INFO) + return logger + +def logger_setup_color(logger, color='auto'): + from bb.msg import BBLogFormatter + console = logging.StreamHandler(sys.stdout) + formatter = BBLogFormatter("%(levelname)s: %(message)s") + console.setFormatter(formatter) + logger.handlers = [console] + if color == 'always' or (color=='auto' and console.stream.isatty()): + formatter.enable_color() + + +def load_plugins(logger, plugins, pluginpath): + import imp + + def load_plugin(name): + logger.debug('Loading plugin %s' % name) + fp, pathname, description = imp.find_module(name, [pluginpath]) + try: + return imp.load_module(name, fp, pathname, description) + finally: + if fp: + fp.close() + + logger.debug('Loading plugins from %s...' % pluginpath) + for fn in glob.glob(os.path.join(pluginpath, '*.py')): + name = os.path.splitext(os.path.basename(fn))[0] + if name != '__init__': + plugin = load_plugin(name) + if hasattr(plugin, 'plugin_init'): + plugin.plugin_init(plugins) + plugins.append(plugin) + +def git_convert_standalone_clone(repodir): + """If specified directory is a git repository, ensure it's a standalone clone""" + import bb.process + if os.path.exists(os.path.join(repodir, '.git')): + alternatesfile = os.path.join(repodir, '.git', 'objects', 'info', 'alternates') + if os.path.exists(alternatesfile): + # This will have been cloned with -s, so we need to convert it so none + # of the contents is shared + bb.process.run('git repack -a', cwd=repodir) + os.remove(alternatesfile) + +def fetch_uri(d, uri, destdir, srcrev=None): + """Fetch a URI to a local directory""" + import bb.data + bb.utils.mkdirhier(destdir) + localdata = bb.data.createCopy(d) + localdata.setVar('BB_STRICT_CHECKSUM', '') + localdata.setVar('SRCREV', srcrev) + ret = (None, None) + olddir = os.getcwd() + try: + fetcher = bb.fetch2.Fetch([uri], localdata) + for u in fetcher.ud: + ud = fetcher.ud[u] + ud.ignore_checksums = True + fetcher.download() + for u in fetcher.ud: + ud = fetcher.ud[u] + if ud.localpath.rstrip(os.sep) == localdata.getVar('DL_DIR', True).rstrip(os.sep): + raise Exception('Local path is download directory - please check that the URI "%s" is correct' % uri) + fetcher.unpack(destdir) + for u in fetcher.ud: + ud = fetcher.ud[u] + if ud.method.recommends_checksum(ud): + md5value = bb.utils.md5_file(ud.localpath) + sha256value = bb.utils.sha256_file(ud.localpath) + ret = (md5value, sha256value) + finally: + os.chdir(olddir) + return ret + +def run_editor(fn): + if isinstance(fn, basestring): + params = '"%s"' % fn + else: + params = '' + for fnitem in fn: + params += ' "%s"' % fnitem + + editor = os.getenv('VISUAL', os.getenv('EDITOR', 'vi')) + try: + return subprocess.check_call('%s %s' % (editor, params), shell=True) + except OSError as exc: + logger.error("Execution of editor '%s' failed: %s", editor, exc) + return 1 diff --git a/scripts/lib/wic/__init__.py b/scripts/lib/wic/__init__.py new file mode 100644 index 0000000..63c1d9c --- /dev/null +++ b/scripts/lib/wic/__init__.py @@ -0,0 +1,4 @@ +import os, sys + +cur_path = os.path.dirname(__file__) or '.' +sys.path.insert(0, cur_path + '/3rdparty') diff --git a/scripts/lib/wic/__version__.py b/scripts/lib/wic/__version__.py new file mode 100644 index 0000000..5452a46 --- /dev/null +++ b/scripts/lib/wic/__version__.py @@ -0,0 +1 @@ +VERSION = "2.00" diff --git a/scripts/lib/wic/canned-wks/common.wks.inc b/scripts/lib/wic/canned-wks/common.wks.inc new file mode 100644 index 0000000..5cf2fd1 --- /dev/null +++ b/scripts/lib/wic/canned-wks/common.wks.inc @@ -0,0 +1,3 @@ +# This file is included into 3 canned wks files from this directory +part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024 +part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 diff --git a/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg b/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg new file mode 100644 index 0000000..a16bd6a --- /dev/null +++ b/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg @@ -0,0 +1,11 @@ +# This is an example configuration file for syslinux. +PROMPT 0 +TIMEOUT 10 + +ALLOWOPTIONS 1 +SERIAL 0 115200 + +DEFAULT boot +LABEL boot +KERNEL /vmlinuz +APPEND label=boot root=/dev/sda2 rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 diff --git a/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks b/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks new file mode 100644 index 0000000..3529e05 --- /dev/null +++ b/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks @@ -0,0 +1,8 @@ +# short-description: Create a 'pcbios' direct disk image with custom bootloader config +# long-description: Creates a partitioned legacy BIOS disk image that the user +# can directly dd to boot media. The bootloader configuration source is a user file. + +include common.wks.inc + +bootloader --configfile="directdisk-bootloader-config.cfg" + diff --git a/scripts/lib/wic/canned-wks/directdisk-gpt.wks b/scripts/lib/wic/canned-wks/directdisk-gpt.wks new file mode 100644 index 0000000..ea01cf3 --- /dev/null +++ b/scripts/lib/wic/canned-wks/directdisk-gpt.wks @@ -0,0 +1,10 @@ +# short-description: Create a 'pcbios' direct disk image +# long-description: Creates a partitioned legacy BIOS disk image that the user +# can directly dd to boot media. + + +part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024 +part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid + +bootloader --ptable gpt --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0" + diff --git a/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks b/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks new file mode 100644 index 0000000..8a81f8f --- /dev/null +++ b/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks @@ -0,0 +1,23 @@ +# short-description: Create multi rootfs image using rootfs plugin +# long-description: Creates a partitioned disk image with two rootfs partitions +# using rootfs plugin. +# +# Partitions can use either +# - indirect rootfs references to image recipe(s): +# wic create directdisk-multi-indirect-recipes -e core-image-minimal \ +# --rootfs-dir rootfs1=core-image-minimal +# --rootfs-dir rootfs2=core-image-minimal-dev +# +# - or paths to rootfs directories: +# wic create directdisk-multi-rootfs \ +# --rootfs-dir rootfs1=tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs/ +# --rootfs-dir rootfs2=tmp/work/qemux86_64-poky-linux/core-image-minimal-dev/1.0-r0/rootfs/ +# +# - or any combinations of -r and --rootfs command line options + +part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024 +part / --source rootfs --rootfs-dir=rootfs1 --ondisk sda --fstype=ext4 --label platform --align 1024 +part /rescue --source rootfs --rootfs-dir=rootfs2 --ondisk sda --fstype=ext4 --label secondary --align 1024 + +bootloader --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0" + diff --git a/scripts/lib/wic/canned-wks/directdisk.wks b/scripts/lib/wic/canned-wks/directdisk.wks new file mode 100644 index 0000000..6db74a7 --- /dev/null +++ b/scripts/lib/wic/canned-wks/directdisk.wks @@ -0,0 +1,8 @@ +# short-description: Create a 'pcbios' direct disk image +# long-description: Creates a partitioned legacy BIOS disk image that the user +# can directly dd to boot media. + +include common.wks.inc + +bootloader --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0" + diff --git a/scripts/lib/wic/canned-wks/mkefidisk.wks b/scripts/lib/wic/canned-wks/mkefidisk.wks new file mode 100644 index 0000000..696e94e --- /dev/null +++ b/scripts/lib/wic/canned-wks/mkefidisk.wks @@ -0,0 +1,11 @@ +# short-description: Create an EFI disk image +# long-description: Creates a partitioned EFI disk image that the user +# can directly dd to boot media. + +part /boot --source bootimg-efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024 + +part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 + +part swap --ondisk sda --size 44 --label swap1 --fstype=swap + +bootloader --timeout=10 --append="rootwait rootfstype=ext4 console=ttyPCH0,115200 console=tty0 vmalloc=256MB snd-hda-intel.enable_msi=0" diff --git a/scripts/lib/wic/canned-wks/mkgummidisk.wks b/scripts/lib/wic/canned-wks/mkgummidisk.wks new file mode 100644 index 0000000..66a22f6 --- /dev/null +++ b/scripts/lib/wic/canned-wks/mkgummidisk.wks @@ -0,0 +1,11 @@ +# short-description: Create an EFI disk image +# long-description: Creates a partitioned EFI disk image that the user +# can directly dd to boot media. + +part /boot --source bootimg-efi --sourceparams="loader=gummiboot" --ondisk sda --label msdos --active --align 1024 + +part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 + +part swap --ondisk sda --size 44 --label swap1 --fstype=swap + +bootloader --timeout=10 --append="rootwait rootfstype=ext4 console=ttyPCH0,115200 console=tty0 vmalloc=256MB snd-hda-intel.enable_msi=0" diff --git a/scripts/lib/wic/canned-wks/mkhybridiso.wks b/scripts/lib/wic/canned-wks/mkhybridiso.wks new file mode 100644 index 0000000..9d34e9b --- /dev/null +++ b/scripts/lib/wic/canned-wks/mkhybridiso.wks @@ -0,0 +1,7 @@ +# short-description: Create a hybrid ISO image +# long-description: Creates an EFI and legacy bootable hybrid ISO image +# which can be used on optical media as well as USB media. + +part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi,image_name=HYBRID_ISO_IMG" --ondisk cd --label HYBRIDISO --fstype=ext4 + +bootloader --timeout=15 --append="" diff --git a/scripts/lib/wic/canned-wks/qemux86-directdisk.wks b/scripts/lib/wic/canned-wks/qemux86-directdisk.wks new file mode 100644 index 0000000..a6518a0 --- /dev/null +++ b/scripts/lib/wic/canned-wks/qemux86-directdisk.wks @@ -0,0 +1,8 @@ +# short-description: Create a qemu machine 'pcbios' direct disk image +# long-description: Creates a partitioned legacy BIOS disk image that the user +# can directly use to boot a qemu machine. + +include common.wks.inc + +bootloader --timeout=0 --append="vga=0 uvesafb.mode_option=640x480-32 root=/dev/vda2 rw mem=256M ip=192.168.7.2::192.168.7.1:255.255.255.0 oprofile.timer=1 rootfstype=ext4 " + diff --git a/scripts/lib/wic/canned-wks/sdimage-bootpart.wks b/scripts/lib/wic/canned-wks/sdimage-bootpart.wks new file mode 100644 index 0000000..7ffd632 --- /dev/null +++ b/scripts/lib/wic/canned-wks/sdimage-bootpart.wks @@ -0,0 +1,6 @@ +# short-description: Create SD card image with a boot partition +# long-description: Creates a partitioned SD card image. Boot files +# are located in the first vfat partition. + +part /boot --source bootimg-partition --ondisk mmcblk --fstype=vfat --label boot --active --align 4 --size 16 +part / --source rootfs --ondisk mmcblk --fstype=ext4 --label root --align 4 diff --git a/scripts/lib/wic/conf.py b/scripts/lib/wic/conf.py new file mode 100644 index 0000000..f7d56d0 --- /dev/null +++ b/scripts/lib/wic/conf.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os + +from wic.ksparser import KickStart, KickStartError +from wic import msger +from wic.utils import misc + + +def get_siteconf(): + wic_path = os.path.dirname(__file__) + eos = wic_path.find('scripts') + len('scripts') + scripts_path = wic_path[:eos] + + return scripts_path + "/lib/image/config/wic.conf" + +class ConfigMgr(object): + DEFAULTS = { + 'common': { + "distro_name": "Default Distribution", + "plugin_dir": "/usr/lib/wic/plugins"}, # TODO use prefix also? + 'create': { + "tmpdir": '/var/tmp/wic', + "outdir": './wic-output', + "release": None, + "logfile": None, + "name_prefix": None, + "name_suffix": None} + } + + # make the manager class as singleton + _instance = None + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(ConfigMgr, cls).__new__(cls, *args, **kwargs) + + return cls._instance + + def __init__(self, ksconf=None, siteconf=None): + # reset config options + self.reset() + + if not siteconf: + siteconf = get_siteconf() + + # initial options from siteconf + self._siteconf = siteconf + + if ksconf: + self._ksconf = ksconf + + def reset(self): + self.__ksconf = None + self.__siteconf = None + self.create = {} + + # initialize the values with defaults + for sec, vals in self.DEFAULTS.iteritems(): + setattr(self, sec, vals) + + def __set_ksconf(self, ksconf): + if not os.path.isfile(ksconf): + msger.error('Cannot find ks file: %s' % ksconf) + + self.__ksconf = ksconf + self._parse_kickstart(ksconf) + def __get_ksconf(self): + return self.__ksconf + _ksconf = property(__get_ksconf, __set_ksconf) + + def _parse_kickstart(self, ksconf=None): + if not ksconf: + return + + try: + ksobj = KickStart(ksconf) + except KickStartError as err: + msger.error(str(err)) + + self.create['ks'] = ksobj + self.create['name'] = os.path.splitext(os.path.basename(ksconf))[0] + + self.create['name'] = misc.build_name(ksconf, + self.create['release'], + self.create['name_prefix'], + self.create['name_suffix']) + +configmgr = ConfigMgr() diff --git a/scripts/lib/wic/config/wic.conf b/scripts/lib/wic/config/wic.conf new file mode 100644 index 0000000..a51bcb5 --- /dev/null +++ b/scripts/lib/wic/config/wic.conf @@ -0,0 +1,6 @@ +[common] +; general settings +distro_name = OpenEmbedded + +[create] +; settings for create subcommand diff --git a/scripts/lib/wic/creator.py b/scripts/lib/wic/creator.py new file mode 100644 index 0000000..5231297 --- /dev/null +++ b/scripts/lib/wic/creator.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os, sys +from optparse import OptionParser, SUPPRESS_HELP + +from wic import msger +from wic.utils import errors +from wic.conf import configmgr +from wic.plugin import pluginmgr + + +class Creator(object): + """${name}: create an image + + Usage: + ${name} SUBCOMMAND <ksfile> [OPTS] + + ${command_list} + ${option_list} + """ + + name = 'wic create(cr)' + + def __init__(self, *args, **kwargs): + self._subcmds = {} + + # get cmds from pluginmgr + # mix-in do_subcmd interface + for subcmd, klass in pluginmgr.get_plugins('imager').iteritems(): + if not hasattr(klass, 'do_create'): + msger.warning("Unsupported subcmd: %s" % subcmd) + continue + + func = getattr(klass, 'do_create') + self._subcmds[subcmd] = func + + def get_optparser(self): + optparser = OptionParser() + optparser.add_option('-d', '--debug', action='store_true', + dest='debug', + help=SUPPRESS_HELP) + optparser.add_option('-v', '--verbose', action='store_true', + dest='verbose', + help=SUPPRESS_HELP) + optparser.add_option('', '--logfile', type='string', dest='logfile', + default=None, + help='Path of logfile') + optparser.add_option('-c', '--config', type='string', dest='config', + default=None, + help='Specify config file for wic') + optparser.add_option('-o', '--outdir', type='string', action='store', + dest='outdir', default=None, + help='Output directory') + optparser.add_option('', '--tmpfs', action='store_true', dest='enabletmpfs', + help='Setup tmpdir as tmpfs to accelerate, experimental' + ' feature, use it if you have more than 4G memory') + return optparser + + def postoptparse(self, options): + abspath = lambda pth: os.path.abspath(os.path.expanduser(pth)) + + if options.verbose: + msger.set_loglevel('verbose') + if options.debug: + msger.set_loglevel('debug') + + if options.logfile: + logfile_abs_path = abspath(options.logfile) + if os.path.isdir(logfile_abs_path): + raise errors.Usage("logfile's path %s should be file" + % options.logfile) + if not os.path.exists(os.path.dirname(logfile_abs_path)): + os.makedirs(os.path.dirname(logfile_abs_path)) + msger.set_interactive(False) + msger.set_logfile(logfile_abs_path) + configmgr.create['logfile'] = options.logfile + + if options.config: + configmgr.reset() + configmgr._siteconf = options.config + + if options.outdir is not None: + configmgr.create['outdir'] = abspath(options.outdir) + + cdir = 'outdir' + if os.path.exists(configmgr.create[cdir]) \ + and not os.path.isdir(configmgr.create[cdir]): + msger.error('Invalid directory specified: %s' \ + % configmgr.create[cdir]) + + if options.enabletmpfs: + configmgr.create['enabletmpfs'] = options.enabletmpfs + + def main(self, argv=None): + if argv is None: + argv = sys.argv + else: + argv = argv[:] # don't modify caller's list + + pname = argv[0] + if pname not in self._subcmds: + msger.error('Unknown plugin: %s' % pname) + + optparser = self.get_optparser() + options, args = optparser.parse_args(argv) + + self.postoptparse(options) + + return self._subcmds[pname](options, *args[1:]) diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py new file mode 100644 index 0000000..76b93e8 --- /dev/null +++ b/scripts/lib/wic/engine.py @@ -0,0 +1,220 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION + +# This module implements the image creation engine used by 'wic' to +# create images. The engine parses through the OpenEmbedded kickstart +# (wks) file specified and generates images that can then be directly +# written onto media. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# + +import os +import sys + +from wic import msger, creator +from wic.utils import misc +from wic.plugin import pluginmgr +from wic.utils.oe import misc + + +def verify_build_env(): + """ + Verify that the build environment is sane. + + Returns True if it is, false otherwise + """ + if not os.environ.get("BUILDDIR"): + print "BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)" + sys.exit(1) + + return True + + +CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts +SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR + +def build_canned_image_list(path): + layers_path = misc.get_bitbake_var("BBLAYERS") + canned_wks_layer_dirs = [] + + if layers_path is not None: + for layer_path in layers_path.split(): + cpath = os.path.join(layer_path, SCRIPTS_CANNED_IMAGE_DIR) + canned_wks_layer_dirs.append(cpath) + + cpath = os.path.join(path, CANNED_IMAGE_DIR) + canned_wks_layer_dirs.append(cpath) + + return canned_wks_layer_dirs + +def find_canned_image(scripts_path, wks_file): + """ + Find a .wks file with the given name in the canned files dir. + + Return False if not found + """ + layers_canned_wks_dir = build_canned_image_list(scripts_path) + + for canned_wks_dir in layers_canned_wks_dir: + for root, dirs, files in os.walk(canned_wks_dir): + for fname in files: + if fname.endswith("~") or fname.endswith("#"): + continue + if fname.endswith(".wks") and wks_file + ".wks" == fname: + fullpath = os.path.join(canned_wks_dir, fname) + return fullpath + return None + + +def list_canned_images(scripts_path): + """ + List the .wks files in the canned image dir, minus the extension. + """ + layers_canned_wks_dir = build_canned_image_list(scripts_path) + + for canned_wks_dir in layers_canned_wks_dir: + for root, dirs, files in os.walk(canned_wks_dir): + for fname in files: + if fname.endswith("~") or fname.endswith("#"): + continue + if fname.endswith(".wks"): + fullpath = os.path.join(canned_wks_dir, fname) + with open(fullpath) as wks: + for line in wks: + desc = "" + idx = line.find("short-description:") + if idx != -1: + desc = line[idx + len("short-description:"):].strip() + break + basename = os.path.splitext(fname)[0] + print " %s\t\t%s" % (basename.ljust(30), desc) + + +def list_canned_image_help(scripts_path, fullpath): + """ + List the help and params in the specified canned image. + """ + found = False + with open(fullpath) as wks: + for line in wks: + if not found: + idx = line.find("long-description:") + if idx != -1: + print + print line[idx + len("long-description:"):].strip() + found = True + continue + if not line.strip(): + break + idx = line.find("#") + if idx != -1: + print line[idx + len("#:"):].rstrip() + else: + break + + +def list_source_plugins(): + """ + List the available source plugins i.e. plugins available for --source. + """ + plugins = pluginmgr.get_source_plugins() + + for plugin in plugins: + print " %s" % plugin + + +def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, + native_sysroot, scripts_path, image_output_dir, + compressor, debug): + """Create image + + wks_file - user-defined OE kickstart file + rootfs_dir - absolute path to the build's /rootfs dir + bootimg_dir - absolute path to the build's boot artifacts directory + kernel_dir - absolute path to the build's kernel directory + native_sysroot - absolute path to the build's native sysroots dir + scripts_path - absolute path to /scripts dir + image_output_dir - dirname to create for image + compressor - compressor utility to compress the image + + Normally, the values for the build artifacts values are determined + by 'wic -e' from the output of the 'bitbake -e' command given an + image name e.g. 'core-image-minimal' and a given machine set in + local.conf. If that's the case, the variables get the following + values from the output of 'bitbake -e': + + rootfs_dir: IMAGE_ROOTFS + kernel_dir: DEPLOY_DIR_IMAGE + native_sysroot: STAGING_DIR_NATIVE + + In the above case, bootimg_dir remains unset and the + plugin-specific image creation code is responsible for finding the + bootimg artifacts. + + In the case where the values are passed in explicitly i.e 'wic -e' + is not used but rather the individual 'wic' options are used to + explicitly specify these values. + """ + try: + oe_builddir = os.environ["BUILDDIR"] + except KeyError: + print "BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)" + sys.exit(1) + + if debug: + msger.set_loglevel('debug') + + crobj = creator.Creator() + + crobj.main(["direct", native_sysroot, kernel_dir, bootimg_dir, rootfs_dir, + wks_file, image_output_dir, oe_builddir, compressor or ""]) + + print "\nThe image(s) were created using OE kickstart file:\n %s" % wks_file + + +def wic_list(args, scripts_path): + """ + Print the list of images or source plugins. + """ + if len(args) < 1: + return False + + if args == ["images"]: + list_canned_images(scripts_path) + return True + elif args == ["source-plugins"]: + list_source_plugins() + return True + elif len(args) == 2 and args[1] == "help": + wks_file = args[0] + fullpath = find_canned_image(scripts_path, wks_file) + if not fullpath: + print "No image named %s found, exiting. "\ + "(Use 'wic list images' to list available images, or "\ + "specify a fully-qualified OE kickstart (.wks) "\ + "filename)\n" % wks_file + sys.exit(1) + list_canned_image_help(scripts_path, fullpath) + return True + + return False diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py new file mode 100644 index 0000000..405d25a --- /dev/null +++ b/scripts/lib/wic/help.py @@ -0,0 +1,777 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This module implements some basic help invocation functions along +# with the bulk of the help topic text for the OE Core Image Tools. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# + +import subprocess +import logging + +from wic.plugin import pluginmgr, PLUGIN_TYPES + +def subcommand_error(args): + logging.info("invalid subcommand %s" % args[0]) + + +def display_help(subcommand, subcommands): + """ + Display help for subcommand. + """ + if subcommand not in subcommands: + return False + + hlp = subcommands.get(subcommand, subcommand_error)[2] + if callable(hlp): + hlp = hlp() + pager = subprocess.Popen('less', stdin=subprocess.PIPE) + pager.communicate(hlp) + + return True + + +def wic_help(args, usage_str, subcommands): + """ + Subcommand help dispatcher. + """ + if len(args) == 1 or not display_help(args[1], subcommands): + print usage_str + + +def get_wic_plugins_help(): + """ + Combine wic_plugins_help with the help for every known + source plugin. + """ + result = wic_plugins_help + for plugin_type in PLUGIN_TYPES: + result += '\n\n%s PLUGINS\n\n' % plugin_type.upper() + for name, plugin in pluginmgr.get_plugins(plugin_type).iteritems(): + result += "\n %s plugin:\n" % name + if plugin.__doc__: + result += plugin.__doc__ + else: + result += "\n %s is missing docstring\n" % plugin + return result + + +def invoke_subcommand(args, parser, main_command_usage, subcommands): + """ + Dispatch to subcommand handler borrowed from combo-layer. + Should use argparse, but has to work in 2.6. + """ + if not args: + logging.error("No subcommand specified, exiting") + parser.print_help() + return 1 + elif args[0] == "help": + wic_help(args, main_command_usage, subcommands) + elif args[0] not in subcommands: + logging.error("Unsupported subcommand %s, exiting\n" % (args[0])) + parser.print_help() + return 1 + else: + usage = subcommands.get(args[0], subcommand_error)[1] + subcommands.get(args[0], subcommand_error)[0](args[1:], usage) + + +## +# wic help and usage strings +## + +wic_usage = """ + + Create a customized OpenEmbedded image + + usage: wic [--version] | [--help] | [COMMAND [ARGS]] + + Current 'wic' commands are: + help Show help for command or one of the topics (see below) + create Create a new OpenEmbedded image + list List available canned images and source plugins + + Help topics: + overview wic overview - General overview of wic + plugins wic plugins - Overview and API + kickstart wic kickstart - wic kickstart reference +""" + +wic_help_usage = """ + + usage: wic help <subcommand> + + This command displays detailed help for the specified subcommand. +""" + +wic_create_usage = """ + + Create a new OpenEmbedded image + + usage: wic create <wks file or image name> [-o <DIRNAME> | --outdir <DIRNAME>] + [-i <JSON PROPERTY FILE> | --infile <JSON PROPERTY_FILE>] + [-e | --image-name] [-s, --skip-build-check] [-D, --debug] + [-r, --rootfs-dir] [-b, --bootimg-dir] + [-k, --kernel-dir] [-n, --native-sysroot] [-f, --build-rootfs] + + This command creates an OpenEmbedded image based on the 'OE kickstart + commands' found in the <wks file>. + + The -o option can be used to place the image in a directory with a + different name and location. + + See 'wic help create' for more detailed instructions. +""" + +wic_create_help = """ + +NAME + wic create - Create a new OpenEmbedded image + +SYNOPSIS + wic create <wks file or image name> [-o <DIRNAME> | --outdir <DIRNAME>] + [-e | --image-name] [-s, --skip-build-check] [-D, --debug] + [-r, --rootfs-dir] [-b, --bootimg-dir] + [-k, --kernel-dir] [-n, --native-sysroot] [-f, --build-rootfs] + [-c, --compress-with] + +DESCRIPTION + This command creates an OpenEmbedded image based on the 'OE + kickstart commands' found in the <wks file>. + + In order to do this, wic needs to know the locations of the + various build artifacts required to build the image. + + Users can explicitly specify the build artifact locations using + the -r, -b, -k, and -n options. See below for details on where + the corresponding artifacts are typically found in a normal + OpenEmbedded build. + + Alternatively, users can use the -e option to have 'wic' determine + those locations for a given image. If the -e option is used, the + user needs to have set the appropriate MACHINE variable in + local.conf, and have sourced the build environment. + + The -e option is used to specify the name of the image to use the + artifacts from e.g. core-image-sato. + + The -r option is used to specify the path to the /rootfs dir to + use as the .wks rootfs source. + + The -b option is used to specify the path to the dir containing + the boot artifacts (e.g. /EFI or /syslinux dirs) to use as the + .wks bootimg source. + + The -k option is used to specify the path to the dir containing + the kernel to use in the .wks bootimg. + + The -n option is used to specify the path to the native sysroot + containing the tools to use to build the image. + + The -f option is used to build rootfs by running "bitbake <image>" + + The -s option is used to skip the build check. The build check is + a simple sanity check used to determine whether the user has + sourced the build environment so that the -e option can operate + correctly. If the user has specified the build artifact locations + explicitly, 'wic' assumes the user knows what he or she is doing + and skips the build check. + + The -D option is used to display debug information detailing + exactly what happens behind the scenes when a create request is + fulfilled (or not, as the case may be). It enumerates and + displays the command sequence used, and should be included in any + bug report describing unexpected results. + + When 'wic -e' is used, the locations for the build artifacts + values are determined by 'wic -e' from the output of the 'bitbake + -e' command given an image name e.g. 'core-image-minimal' and a + given machine set in local.conf. In that case, the image is + created as if the following 'bitbake -e' variables were used: + + -r: IMAGE_ROOTFS + -k: STAGING_KERNEL_DIR + -n: STAGING_DIR_NATIVE + -b: empty (plugin-specific handlers must determine this) + + If 'wic -e' is not used, the user needs to select the appropriate + value for -b (as well as -r, -k, and -n). + + The -o option can be used to place the image in a directory with a + different name and location. + + The -c option is used to specify compressor utility to compress + an image. gzip, bzip2 and xz compressors are supported. +""" + +wic_list_usage = """ + + List available OpenEmbedded images and source plugins + + usage: wic list images + wic list <image> help + wic list source-plugins + + This command enumerates the set of available canned images as well as + help for those images. It also can be used to list of available source + plugins. + + The first form enumerates all the available 'canned' images. + + The second form lists the detailed help information for a specific + 'canned' image. + + The third form enumerates all the available --sources (source + plugins). + + See 'wic help list' for more details. +""" + +wic_list_help = """ + +NAME + wic list - List available OpenEmbedded images and source plugins + +SYNOPSIS + wic list images + wic list <image> help + wic list source-plugins + +DESCRIPTION + This command enumerates the set of available canned images as well + as help for those images. It also can be used to list available + source plugins. + + The first form enumerates all the available 'canned' images. + These are actually just the set of .wks files that have been moved + into the /scripts/lib/wic/canned-wks directory). + + The second form lists the detailed help information for a specific + 'canned' image. + + The third form enumerates all the available --sources (source + plugins). The contents of a given partition are driven by code + defined in 'source plugins'. Users specify a specific plugin via + the --source parameter of the partition .wks command. Normally + this is the 'rootfs' plugin but can be any of the more specialized + sources listed by the 'list source-plugins' command. Users can + also add their own source plugins - see 'wic help plugins' for + details. +""" + +wic_plugins_help = """ + +NAME + wic plugins - Overview and API + +DESCRIPTION + plugins allow wic functionality to be extended and specialized by + users. This section documents the plugin interface, which is + currently restricted to 'source' plugins. + + 'Source' plugins provide a mechanism to customize various aspects + of the image generation process in wic, mainly the contents of + partitions. + + Source plugins provide a mechanism for mapping values specified in + .wks files using the --source keyword to a particular plugin + implementation that populates a corresponding partition. + + A source plugin is created as a subclass of SourcePlugin (see + scripts/lib/wic/pluginbase.py) and the plugin file containing it + is added to scripts/lib/wic/plugins/source/ to make the plugin + implementation available to the wic implementation. + + Source plugins can also be implemented and added by external + layers - any plugins found in a scripts/lib/wic/plugins/source/ + directory in an external layer will also be made available. + + When the wic implementation needs to invoke a partition-specific + implementation, it looks for the plugin that has the same name as + the --source param given to that partition. For example, if the + partition is set up like this: + + part /boot --source bootimg-pcbios ... + + then the methods defined as class members of the plugin having the + matching bootimg-pcbios .name class member would be used. + + To be more concrete, here's the plugin definition that would match + a '--source bootimg-pcbios' usage, along with an example method + that would be called by the wic implementation when it needed to + invoke an implementation-specific partition-preparation function: + + class BootimgPcbiosPlugin(SourcePlugin): + name = 'bootimg-pcbios' + + @classmethod + def do_prepare_partition(self, part, ...) + + If the subclass itself doesn't implement a function, a 'default' + version in a superclass will be located and used, which is why all + plugins must be derived from SourcePlugin. + + The SourcePlugin class defines the following methods, which is the + current set of methods that can be implemented/overridden by + --source plugins. Any methods not implemented by a SourcePlugin + subclass inherit the implementations present in the SourcePlugin + class (see the SourcePlugin source for details): + + do_prepare_partition() + Called to do the actual content population for a + partition. In other words, it 'prepares' the final partition + image which will be incorporated into the disk image. + + do_configure_partition() + Called before do_prepare_partition(), typically used to + create custom configuration files for a partition, for + example syslinux or grub config files. + + do_install_disk() + Called after all partitions have been prepared and assembled + into a disk image. This provides a hook to allow + finalization of a disk image, for example to write an MBR to + it. + + do_stage_partition() + Special content-staging hook called before + do_prepare_partition(), normally empty. + + Typically, a partition will just use the passed-in + parameters, for example the unmodified value of bootimg_dir. + In some cases however, things may need to be more tailored. + As an example, certain files may additionally need to be + take from bootimg_dir + /boot. This hook allows those files + to be staged in a customized fashion. Note that + get_bitbake_var() allows you to access non-standard + variables that you might want to use for these types of + situations. + + This scheme is extensible - adding more hooks is a simple matter + of adding more plugin methods to SourcePlugin and derived classes. + The code that then needs to call the plugin methods uses + plugin.get_source_plugin_methods() to find the method(s) needed by + the call; this is done by filling up a dict with keys containing + the method names of interest - on success, these will be filled in + with the actual methods. Please see the implementation for + examples and details. +""" + +wic_overview_help = """ + +NAME + wic overview - General overview of wic + +DESCRIPTION + The 'wic' command generates partitioned images from existing + OpenEmbedded build artifacts. Image generation is driven by + partitioning commands contained in an 'Openembedded kickstart' + (.wks) file (see 'wic help kickstart') specified either directly + on the command-line or as one of a selection of canned .wks files + (see 'wic list images'). When applied to a given set of build + artifacts, the result is an image or set of images that can be + directly written onto media and used on a particular system. + + The 'wic' command and the infrastructure it's based on is by + definition incomplete - its purpose is to allow the generation of + customized images, and as such was designed to be completely + extensible via a plugin interface (see 'wic help plugins'). + + Background and Motivation + + wic is meant to be a completely independent standalone utility + that initially provides easier-to-use and more flexible + replacements for a couple bits of existing functionality in + oe-core: directdisk.bbclass and mkefidisk.sh. The difference + between wic and those examples is that with wic the functionality + of those scripts is implemented by a general-purpose partitioning + 'language' based on Redhat kickstart syntax). + + The initial motivation and design considerations that lead to the + current tool are described exhaustively in Yocto Bug #3847 + (https://bugzilla.yoctoproject.org/show_bug.cgi?id=3847). + + Implementation and Examples + + wic can be used in two different modes, depending on how much + control the user needs in specifying the Openembedded build + artifacts that will be used in creating the image: 'raw' and + 'cooked'. + + If used in 'raw' mode, artifacts are explicitly specified via + command-line arguments (see example below). + + The more easily usable 'cooked' mode uses the current MACHINE + setting and a specified image name to automatically locate the + artifacts used to create the image. + + OE kickstart files (.wks) can of course be specified directly on + the command-line, but the user can also choose from a set of + 'canned' .wks files available via the 'wic list images' command + (example below). + + In any case, the prerequisite for generating any image is to have + the build artifacts already available. The below examples assume + the user has already build a 'core-image-minimal' for a specific + machine (future versions won't require this redundant step, but + for now that's typically how build artifacts get generated). + + The other prerequisite is to source the build environment: + + $ source oe-init-build-env + + To start out with, we'll generate an image from one of the canned + .wks files. The following generates a list of availailable + images: + + $ wic list images + mkefidisk Create an EFI disk image + directdisk Create a 'pcbios' direct disk image + + You can get more information about any of the available images by + typing 'wic list xxx help', where 'xxx' is one of the image names: + + $ wic list mkefidisk help + + Creates a partitioned EFI disk image that the user can directly dd + to boot media. + + At any time, you can get help on the 'wic' command or any + subcommand (currently 'list' and 'create'). For instance, to get + the description of 'wic create' command and its parameters: + + $ wic create + + Usage: + + Create a new OpenEmbedded image + + usage: wic create <wks file or image name> [-o <DIRNAME> | ...] + [-i <JSON PROPERTY FILE> | --infile <JSON PROPERTY_FILE>] + [-e | --image-name] [-s, --skip-build-check] [-D, --debug] + [-r, --rootfs-dir] [-b, --bootimg-dir] [-k, --kernel-dir] + [-n, --native-sysroot] [-f, --build-rootfs] + + This command creates an OpenEmbedded image based on the 'OE + kickstart commands' found in the <wks file>. + + The -o option can be used to place the image in a directory + with a different name and location. + + See 'wic help create' for more detailed instructions. + ... + + As mentioned in the command, you can get even more detailed + information by adding 'help' to the above: + + $ wic help create + + So, the easiest way to create an image is to use the -e option + with a canned .wks file. To use the -e option, you need to + specify the image used to generate the artifacts and you actually + need to have the MACHINE used to build them specified in your + local.conf (these requirements aren't necessary if you aren't + using the -e options.) Below, we generate a directdisk image, + pointing the process at the core-image-minimal artifacts for the + current MACHINE: + + $ wic create directdisk -e core-image-minimal + + Checking basic build environment... + Done. + + Creating image(s)... + + Info: The new image(s) can be found here: + /var/tmp/wic/build/directdisk-201309252350-sda.direct + + The following build artifacts were used to create the image(s): + + ROOTFS_DIR: ... + BOOTIMG_DIR: ... + KERNEL_DIR: ... + NATIVE_SYSROOT: ... + + The image(s) were created using OE kickstart file: + .../scripts/lib/wic/canned-wks/directdisk.wks + + The output shows the name and location of the image created, and + so that you know exactly what was used to generate the image, each + of the artifacts and the kickstart file used. + + Similarly, you can create a 'mkefidisk' image in the same way + (notice that this example uses a different machine - because it's + using the -e option, you need to change the MACHINE in your + local.conf): + + $ wic create mkefidisk -e core-image-minimal + Checking basic build environment... + Done. + + Creating image(s)... + + Info: The new image(s) can be found here: + /var/tmp/wic/build/mkefidisk-201309260027-sda.direct + + ... + + Here's an example that doesn't take the easy way out and manually + specifies each build artifact, along with a non-canned .wks file, + and also uses the -o option to have wic create the output + somewhere other than the default /var/tmp/wic: + + $ wic create ./test.wks -o ./out --rootfs-dir + tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs + --bootimg-dir tmp/sysroots/qemux86-64/usr/share + --kernel-dir tmp/deploy/images/qemux86-64 + --native-sysroot tmp/sysroots/x86_64-linux + + Creating image(s)... + + Info: The new image(s) can be found here: + out/build/test-201507211313-sda.direct + + The following build artifacts were used to create the image(s): + ROOTFS_DIR: tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs + BOOTIMG_DIR: tmp/sysroots/qemux86-64/usr/share + KERNEL_DIR: tmp/deploy/images/qemux86-64 + NATIVE_SYSROOT: tmp/sysroots/x86_64-linux + + The image(s) were created using OE kickstart file: + ./test.wks + + Here is a content of test.wks: + + part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024 + part / --source rootfs --ondisk sda --fstype=ext3 --label platform --align 1024 + + bootloader --timeout=0 --append="rootwait rootfstype=ext3 video=vesafb vga=0x318 console=tty0" + + + Finally, here's an example of the actual partition language + commands used to generate the mkefidisk image i.e. these are the + contents of the mkefidisk.wks OE kickstart file: + + # short-description: Create an EFI disk image + # long-description: Creates a partitioned EFI disk image that the user + # can directly dd to boot media. + + part /boot --source bootimg-efi --ondisk sda --fstype=efi --active + + part / --source rootfs --ondisk sda --fstype=ext3 --label platform + + part swap --ondisk sda --size 44 --label swap1 --fstype=swap + + bootloader --timeout=10 --append="rootwait console=ttyPCH0,115200" + + You can get a complete listing and description of all the + kickstart commands available for use in .wks files from 'wic help + kickstart'. +""" + +wic_kickstart_help = """ + +NAME + wic kickstart - wic kickstart reference + +DESCRIPTION + This section provides the definitive reference to the wic + kickstart language. It also provides documentation on the list of + --source plugins available for use from the 'part' command (see + the 'Platform-specific Plugins' section below). + + The current wic implementation supports only the basic kickstart + partitioning commands: partition (or part for short) and + bootloader. + + The following is a listing of the commands, their syntax, and + meanings. The commands are based on the Fedora kickstart + documentation but with modifications to reflect wic capabilities. + + http://fedoraproject.org/wiki/Anaconda/Kickstart#part_or_partition + http://fedoraproject.org/wiki/Anaconda/Kickstart#bootloader + + Commands + + * 'part' or 'partition' + + This command creates a partition on the system and uses the + following syntax: + + part [<mountpoint>] + + The <mountpoint> is where the partition will be mounted and + must take of one of the following forms: + + /<path>: For example: /, /usr, or /home + + swap: The partition will be used as swap space. + + If a <mountpoint> is not specified the partition will be created + but will not be mounted. + + Partitions with a <mountpoint> specified will be automatically mounted. + This is achieved by wic adding entries to the fstab during image + generation. In order for a valid fstab to be generated one of the + --ondrive, --ondisk or --use-uuid partition options must be used for + each partition that specifies a mountpoint. + + + The following are supported 'part' options: + + --size: The minimum partition size. Specify an integer value + such as 500. Multipliers k, M ang G can be used. If + not specified, the size is in MB. + You do not need this option if you use --source. + + --source: This option is a wic-specific option that names the + source of the data that will populate the + partition. The most common value for this option + is 'rootfs', but can be any value which maps to a + valid 'source plugin' (see 'wic help plugins'). + + If '--source rootfs' is used, it tells the wic + command to create a partition as large as needed + and to fill it with the contents of the root + filesystem pointed to by the '-r' wic command-line + option (or the equivalent rootfs derived from the + '-e' command-line option). The filesystem type + that will be used to create the partition is driven + by the value of the --fstype option specified for + the partition (see --fstype below). + + If --source <plugin-name>' is used, it tells the + wic command to create a partition as large as + needed and to fill with the contents of the + partition that will be generated by the specified + plugin name using the data pointed to by the '-r' + wic command-line option (or the equivalent rootfs + derived from the '-e' command-line option). + Exactly what those contents and filesystem type end + up being are dependent on the given plugin + implementation. + + If --source option is not used, the wic command + will create empty partition. --size parameter has + to be used to specify size of empty partition. + + --ondisk or --ondrive: Forces the partition to be created on + a particular disk. + + --fstype: Sets the file system type for the partition. These + apply to partitions created using '--source rootfs' (see + --source above). Valid values are: + + ext2 + ext3 + ext4 + btrfs + squashfs + swap + + --fsoptions: Specifies a free-form string of options to be + used when mounting the filesystem. This string + will be copied into the /etc/fstab file of the + installed system and should be enclosed in + quotes. If not specified, the default string is + "defaults". + + --label label: Specifies the label to give to the filesystem + to be made on the partition. If the given + label is already in use by another filesystem, + a new label is created for the partition. + + --active: Marks the partition as active. + + --align (in KBytes): This option is specific to wic and says + to start a partition on an x KBytes + boundary. + + --no-table: This option is specific to wic. Space will be + reserved for the partition and it will be + populated but it will not be added to the + partition table. It may be useful for + bootloaders. + + --extra-space: This option is specific to wic. It adds extra + space after the space filled by the content + of the partition. The final size can go + beyond the size specified by --size. + By default, 10MB. + + --overhead-factor: This option is specific to wic. The + size of the partition is multiplied by + this factor. It has to be greater than or + equal to 1. + The default value is 1.3. + + --part-type: This option is specific to wic. It specifies partition + type GUID for GPT partitions. + List of partition type GUIDS can be found here: + http://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs + + --use-uuid: This option is specific to wic. It makes wic to generate + random globally unique identifier (GUID) for the partition + and use it in bootloader configuration to specify root partition. + + --uuid: This option is specific to wic. It specifies partition UUID. + It's useful if preconfigured partition UUID is added to kernel command line + in bootloader configuration before running wic. In this case .wks file can + be generated or modified to set preconfigured parition UUID using this option. + + * bootloader + + This command allows the user to specify various bootloader + options. The following are supported 'bootloader' options: + + --timeout: Specifies the number of seconds before the + bootloader times out and boots the default option. + + --append: Specifies kernel parameters. These will be added to + bootloader command-line - for example, the syslinux + APPEND or grub kernel command line. + + --configfile: Specifies a user defined configuration file for + the bootloader. This file must be located in the + canned-wks folder or could be the full path to the + file. Using this option will override any other + bootloader option. + + Note that bootloader functionality and boot partitions are + implemented by the various --source plugins that implement + bootloader functionality; the bootloader command essentially + provides a means of modifying bootloader configuration. + + * include + + This command allows the user to include the content of .wks file + into original .wks file. + + Command uses the following syntax: + + include <file> + + The <file> is either path to the file or its name. If name is + specified wic will try to find file in the directories with canned + .wks files. + +""" diff --git a/scripts/lib/wic/imager/__init__.py b/scripts/lib/wic/imager/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/scripts/lib/wic/imager/__init__.py diff --git a/scripts/lib/wic/imager/baseimager.py b/scripts/lib/wic/imager/baseimager.py new file mode 100644 index 0000000..760cf8a --- /dev/null +++ b/scripts/lib/wic/imager/baseimager.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python -tt +# +# Copyright (c) 2007 Red Hat Inc. +# Copyright (c) 2009, 2010, 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +from __future__ import with_statement +import os +import tempfile +import shutil + +from wic import msger +from wic.utils.errors import CreatorError +from wic.utils import runner + +class BaseImageCreator(object): + """Base class for image creation. + + BaseImageCreator is the simplest creator class available; it will + create a system image according to the supplied kickstart file. + + e.g. + + import wic.imgcreate as imgcreate + ks = imgcreate.read_kickstart("foo.ks") + imgcreate.ImageCreator(ks, "foo").create() + """ + + def __del__(self): + self.cleanup() + + def __init__(self, createopts=None): + """Initialize an ImageCreator instance. + + ks -- a pykickstart.KickstartParser instance; this instance will be + used to drive the install by e.g. providing the list of packages + to be installed, the system configuration and %post scripts + + name -- a name for the image; used for e.g. image filenames or + filesystem labels + """ + + self.__builddir = None + + self.ks = None + self.name = "target" + self.tmpdir = "/var/tmp/wic" + self.workdir = "/var/tmp/wic/build" + + # setup tmpfs tmpdir when enabletmpfs is True + self.enabletmpfs = False + + if createopts: + # Mapping table for variables that have different names. + optmap = {"outdir" : "destdir", + } + + # update setting from createopts + for key in createopts.keys(): + if key in optmap: + option = optmap[key] + else: + option = key + setattr(self, option, createopts[key]) + + self.destdir = os.path.abspath(os.path.expanduser(self.destdir)) + + self._dep_checks = ["ls", "bash", "cp", "echo"] + + # Output image file names + self.outimage = [] + + # No ks provided when called by convertor, so skip the dependency check + if self.ks: + # If we have btrfs partition we need to check necessary tools + for part in self.ks.partitions: + if part.fstype and part.fstype == "btrfs": + self._dep_checks.append("mkfs.btrfs") + break + + # make sure the specified tmpdir and cachedir exist + if not os.path.exists(self.tmpdir): + os.makedirs(self.tmpdir) + + + # + # Hooks for subclasses + # + def _create(self): + """Create partitions for the disk image(s) + + This is the hook where subclasses may create the partitions + that will be assembled into disk image(s). + + There is no default implementation. + """ + pass + + def _cleanup(self): + """Undo anything performed in _create(). + + This is the hook where subclasses must undo anything which was + done in _create(). + + There is no default implementation. + + """ + pass + + # + # Actual implementation + # + def __ensure_builddir(self): + if not self.__builddir is None: + return + + try: + self.workdir = os.path.join(self.tmpdir, "build") + if not os.path.exists(self.workdir): + os.makedirs(self.workdir) + self.__builddir = tempfile.mkdtemp(dir=self.workdir, + prefix="imgcreate-") + except OSError as err: + raise CreatorError("Failed create build directory in %s: %s" % + (self.tmpdir, err)) + + def __setup_tmpdir(self): + if not self.enabletmpfs: + return + + runner.show('mount -t tmpfs -o size=4G tmpfs %s' % self.workdir) + + def __clean_tmpdir(self): + if not self.enabletmpfs: + return + + runner.show('umount -l %s' % self.workdir) + + def create(self): + """Create partitions for the disk image(s) + + Create the partitions that will be assembled into disk + image(s). + """ + self.__setup_tmpdir() + self.__ensure_builddir() + + self._create() + + def cleanup(self): + """Undo anything performed in create(). + + Note, make sure to call this method once finished with the creator + instance in order to ensure no stale files are left on the host e.g.: + + creator = ImageCreator(ks, name) + try: + creator.create() + finally: + creator.cleanup() + + """ + if not self.__builddir: + return + + self._cleanup() + + shutil.rmtree(self.__builddir, ignore_errors=True) + self.__builddir = None + + self.__clean_tmpdir() + + + def print_outimage_info(self): + msg = "The new image can be found here:\n" + self.outimage.sort() + for path in self.outimage: + msg += ' %s\n' % os.path.abspath(path) + + msger.info(msg) diff --git a/scripts/lib/wic/imager/direct.py b/scripts/lib/wic/imager/direct.py new file mode 100644 index 0000000..a1b4249 --- /dev/null +++ b/scripts/lib/wic/imager/direct.py @@ -0,0 +1,380 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This implements the 'direct' image creator class for 'wic' +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# + +import os +import shutil + +from wic import msger +from wic.utils import fs_related +from wic.utils.oe.misc import get_bitbake_var +from wic.utils.partitionedfs import Image +from wic.utils.errors import CreatorError, ImageError +from wic.imager.baseimager import BaseImageCreator +from wic.plugin import pluginmgr +from wic.utils.oe.misc import exec_cmd + +disk_methods = { + "do_install_disk":None, +} + +class DirectImageCreator(BaseImageCreator): + """ + Installs a system into a file containing a partitioned disk image. + + DirectImageCreator is an advanced ImageCreator subclass; an image + file is formatted with a partition table, each partition created + from a rootfs or other OpenEmbedded build artifact and dd'ed into + the virtual disk. The disk image can subsequently be dd'ed onto + media and used on actual hardware. + """ + + def __init__(self, oe_builddir, image_output_dir, rootfs_dir, bootimg_dir, + kernel_dir, native_sysroot, compressor, creatoropts=None): + """ + Initialize a DirectImageCreator instance. + + This method takes the same arguments as ImageCreator.__init__() + """ + BaseImageCreator.__init__(self, creatoropts) + + self.__image = None + self.__disks = {} + self.__disk_format = "direct" + self._disk_names = [] + self.ptable_format = self.ks.bootloader.ptable + + self.oe_builddir = oe_builddir + if image_output_dir: + self.tmpdir = image_output_dir + self.rootfs_dir = rootfs_dir + self.bootimg_dir = bootimg_dir + self.kernel_dir = kernel_dir + self.native_sysroot = native_sysroot + self.compressor = compressor + + def __get_part_num(self, num, parts): + """calculate the real partition number, accounting for partitions not + in the partition table and logical partitions + """ + realnum = 0 + for pnum, part in enumerate(parts, 1): + if not part.no_table: + realnum += 1 + if pnum == num: + if part.no_table: + return 0 + if self.ptable_format == 'msdos' and realnum > 3: + # account for logical partition numbering, ex. sda5.. + return realnum + 1 + return realnum + + def _write_fstab(self, image_rootfs): + """overriden to generate fstab (temporarily) in rootfs. This is called + from _create, make sure it doesn't get called from + BaseImage.create() + """ + if not image_rootfs: + return + + fstab_path = image_rootfs + "/etc/fstab" + if not os.path.isfile(fstab_path): + return + + with open(fstab_path) as fstab: + fstab_lines = fstab.readlines() + + if self._update_fstab(fstab_lines, self._get_parts()): + shutil.copyfile(fstab_path, fstab_path + ".orig") + + with open(fstab_path, "w") as fstab: + fstab.writelines(fstab_lines) + + return fstab_path + + def _update_fstab(self, fstab_lines, parts): + """Assume partition order same as in wks""" + updated = False + for num, part in enumerate(parts, 1): + pnum = self.__get_part_num(num, parts) + if not pnum or not part.mountpoint \ + or part.mountpoint in ("/", "/boot"): + continue + + # mmc device partitions are named mmcblk0p1, mmcblk0p2.. + prefix = 'p' if part.disk.startswith('mmcblk') else '' + device_name = "/dev/%s%s%d" % (part.disk, prefix, pnum) + + opts = part.fsopts if part.fsopts else "defaults" + line = "\t".join([device_name, part.mountpoint, part.fstype, + opts, "0", "0"]) + "\n" + + fstab_lines.append(line) + updated = True + + return updated + + def set_bootimg_dir(self, bootimg_dir): + """ + Accessor for bootimg_dir, the actual location used for the source + of the bootimg. Should be set by source plugins (only if they + change the default bootimg source) so the correct info gets + displayed for print_outimage_info(). + """ + self.bootimg_dir = bootimg_dir + + def _get_parts(self): + if not self.ks: + raise CreatorError("Failed to get partition info, " + "please check your kickstart setting.") + + # Set a default partition if no partition is given out + if not self.ks.partitions: + partstr = "part / --size 1900 --ondisk sda --fstype=ext3" + args = partstr.split() + part = self.ks.parse(args[1:]) + if part not in self.ks.partitions: + self.ks.partitions.append(part) + + # partitions list from kickstart file + return self.ks.partitions + + def get_disk_names(self): + """ Returns a list of physical target disk names (e.g., 'sdb') which + will be created. """ + + if self._disk_names: + return self._disk_names + + #get partition info from ks handler + parts = self._get_parts() + + for i in range(len(parts)): + if parts[i].disk: + disk_name = parts[i].disk + else: + raise CreatorError("Failed to create disks, no --ondisk " + "specified in partition line of ks file") + + if parts[i].mountpoint and not parts[i].fstype: + raise CreatorError("Failed to create disks, no --fstype " + "specified for partition with mountpoint " + "'%s' in the ks file") + + self._disk_names.append(disk_name) + + return self._disk_names + + def _full_name(self, name, extention): + """ Construct full file name for a file we generate. """ + return "%s-%s.%s" % (self.name, name, extention) + + def _full_path(self, path, name, extention): + """ Construct full file path to a file we generate. """ + return os.path.join(path, self._full_name(name, extention)) + + def get_default_source_plugin(self): + """ + The default source plugin i.e. the plugin that's consulted for + overall image generation tasks outside of any particular + partition. For convenience, we just hang it off the + bootloader handler since it's the one non-partition object in + any setup. By default the default plugin is set to the same + plugin as the /boot partition; since we hang it off the + bootloader object, the default can be explicitly set using the + --source bootloader param. + """ + return self.ks.bootloader.source + + # + # Actual implemention + # + def _create(self): + """ + For 'wic', we already have our build artifacts - we just create + filesystems from the artifacts directly and combine them into + a partitioned image. + """ + parts = self._get_parts() + + self.__image = Image(self.native_sysroot) + + for part in parts: + # as a convenience, set source to the boot partition source + # instead of forcing it to be set via bootloader --source + if not self.ks.bootloader.source and part.mountpoint == "/boot": + self.ks.bootloader.source = part.source + + fstab_path = self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR")) + + shutil.rmtree(self.workdir) + os.mkdir(self.workdir) + + for part in parts: + # get rootfs size from bitbake variable if it's not set in .ks file + if not part.size: + # and if rootfs name is specified for the partition + image_name = part.rootfs_dir + if image_name: + # Bitbake variable ROOTFS_SIZE is calculated in + # Image._get_rootfs_size method from meta/lib/oe/image.py + # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, + # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE + rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name) + if rsize_bb: + part.size = int(round(float(rsize_bb))) + # need to create the filesystems in order to get their + # sizes before we can add them and do the layout. + # Image.create() actually calls __format_disks() to create + # the disk images and carve out the partitions, then + # self.assemble() calls Image.assemble() which calls + # __write_partitition() for each partition to dd the fs + # into the partitions. + part.prepare(self, self.workdir, self.oe_builddir, self.rootfs_dir, + self.bootimg_dir, self.kernel_dir, self.native_sysroot) + + + self.__image.add_partition(int(part.size), + part.disk, + part.mountpoint, + part.source_file, + part.fstype, + part.label, + fsopts=part.fsopts, + boot=part.active, + align=part.align, + no_table=part.no_table, + part_type=part.part_type, + uuid=part.uuid) + + if fstab_path: + shutil.move(fstab_path + ".orig", fstab_path) + + self.__image.layout_partitions(self.ptable_format) + + self.__imgdir = self.workdir + for disk_name, disk in self.__image.disks.items(): + full_path = self._full_path(self.__imgdir, disk_name, "direct") + msger.debug("Adding disk %s as %s with size %s bytes" \ + % (disk_name, full_path, disk['min_size'])) + disk_obj = fs_related.DiskImage(full_path, disk['min_size']) + self.__disks[disk_name] = disk_obj + self.__image.add_disk(disk_name, disk_obj) + + self.__image.create() + + def assemble(self): + """ + Assemble partitions into disk image(s) + """ + for disk_name, disk in self.__image.disks.items(): + full_path = self._full_path(self.__imgdir, disk_name, "direct") + msger.debug("Assembling disk %s as %s with size %s bytes" \ + % (disk_name, full_path, disk['min_size'])) + self.__image.assemble(full_path) + + def finalize(self): + """ + Finalize the disk image. + + For example, prepare the image to be bootable by e.g. + creating and installing a bootloader configuration. + + """ + source_plugin = self.get_default_source_plugin() + if source_plugin: + self._source_methods = pluginmgr.get_source_plugin_methods(source_plugin, disk_methods) + for disk_name, disk in self.__image.disks.items(): + self._source_methods["do_install_disk"](disk, disk_name, self, + self.workdir, + self.oe_builddir, + self.bootimg_dir, + self.kernel_dir, + self.native_sysroot) + # Compress the image + if self.compressor: + for disk_name, disk in self.__image.disks.items(): + full_path = self._full_path(self.__imgdir, disk_name, "direct") + msger.debug("Compressing disk %s with %s" % \ + (disk_name, self.compressor)) + exec_cmd("%s %s" % (self.compressor, full_path)) + + def print_outimage_info(self): + """ + Print the image(s) and artifacts used, for the user. + """ + msg = "The new image(s) can be found here:\n" + + parts = self._get_parts() + + for disk_name in self.__image.disks: + extension = "direct" + {"gzip": ".gz", + "bzip2": ".bz2", + "xz": ".xz", + "": ""}.get(self.compressor) + full_path = self._full_path(self.__imgdir, disk_name, extension) + msg += ' %s\n\n' % full_path + + msg += 'The following build artifacts were used to create the image(s):\n' + for part in parts: + if part.rootfs_dir is None: + continue + if part.mountpoint == '/': + suffix = ':' + else: + suffix = '["%s"]:' % (part.mountpoint or part.label) + msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), part.rootfs_dir) + + msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir + msg += ' KERNEL_DIR: %s\n' % self.kernel_dir + msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot + + msger.info(msg) + + @property + def rootdev(self): + """ + Get root device name to use as a 'root' parameter + in kernel command line. + + Assume partition order same as in wks + """ + parts = self._get_parts() + for num, part in enumerate(parts, 1): + if part.mountpoint == "/": + if part.uuid: + return "PARTUUID=%s" % part.uuid + else: + suffix = 'p' if part.disk.startswith('mmcblk') else '' + pnum = self.__get_part_num(num, parts) + return "/dev/%s%s%-d" % (part.disk, suffix, pnum) + + def _cleanup(self): + if not self.__image is None: + try: + self.__image.cleanup() + except ImageError, err: + msger.warning("%s" % err) + diff --git a/scripts/lib/wic/ksparser.py b/scripts/lib/wic/ksparser.py new file mode 100644 index 0000000..8c3f808 --- /dev/null +++ b/scripts/lib/wic/ksparser.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python -tt +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2016 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# DESCRIPTION +# This module provides parser for kickstart format +# +# AUTHORS +# Ed Bartosh <ed.bartosh> (at] linux.intel.com> + +"""Kickstart parser module.""" + +import os +import shlex +from argparse import ArgumentParser, ArgumentError, ArgumentTypeError + +from wic import msger +from wic.partition import Partition +from wic.utils.misc import find_canned + +class KickStartError(Exception): + """Custom exception.""" + pass + +class KickStartParser(ArgumentParser): + """ + This class overwrites error method to throw exception + instead of producing usage message(default argparse behavior). + """ + def error(self, message): + raise ArgumentError(None, message) + +def sizetype(arg): + """ + Custom type for ArgumentParser + Converts size string in <num>[K|k|M|G] format into the integer value + """ + if arg.isdigit(): + return int(arg) * 1024L + + if not arg[:-1].isdigit(): + raise ArgumentTypeError("Invalid size: %r" % arg) + + size = int(arg[:-1]) + if arg.endswith("k") or arg.endswith("K"): + return size + if arg.endswith("M"): + return size * 1024L + if arg.endswith("G"): + return size * 1024L * 1024L + + raise ArgumentTypeError("Invalid size: %r" % arg) + +def overheadtype(arg): + """ + Custom type for ArgumentParser + Converts overhead string to float and checks if it's bigger than 1.0 + """ + try: + result = float(arg) + except ValueError: + raise ArgumentTypeError("Invalid value: %r" % arg) + + if result < 1.0: + raise ArgumentTypeError("Overhead factor should be > 1.0" % arg) + + return result + +def cannedpathtype(arg): + """ + Custom type for ArgumentParser + Tries to find file in the list of canned wks paths + """ + scripts_path = os.path.abspath(os.path.dirname(__file__) + '../../..') + result = find_canned(scripts_path, arg) + if not result: + raise ArgumentTypeError("file not found: %s" % arg) + return result + +class KickStart(object): + """"Kickstart parser implementation.""" + + def __init__(self, confpath): + + self.partitions = [] + self.bootloader = None + self.lineno = 0 + self.partnum = 0 + + parser = KickStartParser() + subparsers = parser.add_subparsers() + + part = subparsers.add_parser('part') + part.add_argument('mountpoint') + part.add_argument('--active', action='store_true') + part.add_argument('--align', type=int) + part.add_argument("--extra-space", type=sizetype, default=10*1024L) + part.add_argument('--fsoptions', dest='fsopts') + part.add_argument('--fstype') + part.add_argument('--label') + part.add_argument('--no-table', action='store_true') + part.add_argument('--ondisk', '--ondrive', dest='disk') + part.add_argument("--overhead-factor", type=overheadtype, default=1.3) + part.add_argument('--part-type') + part.add_argument('--rootfs-dir') + part.add_argument('--size', type=sizetype, default=0) + part.add_argument('--source') + part.add_argument('--sourceparams') + part.add_argument('--use-uuid', action='store_true') + part.add_argument('--uuid') + + bootloader = subparsers.add_parser('bootloader') + bootloader.add_argument('--append') + bootloader.add_argument('--configfile') + bootloader.add_argument('--ptable', choices=('msdos', 'gpt'), + default='msdos') + bootloader.add_argument('--timeout', type=int) + bootloader.add_argument('--source') + + include = subparsers.add_parser('include') + include.add_argument('path', type=cannedpathtype) + + self._parse(parser, confpath) + if not self.bootloader: + msger.warning('bootloader config not specified, using defaults') + self.bootloader = bootloader.parse_args([]) + + def _parse(self, parser, confpath): + """ + Parse file in .wks format using provided parser. + """ + with open(confpath) as conf: + lineno = 0 + for line in conf: + line = line.strip() + lineno += 1 + if line and line[0] != '#': + try: + parsed = parser.parse_args(shlex.split(line)) + except ArgumentError as err: + raise KickStartError('%s:%d: %s' % \ + (confpath, lineno, err)) + if line.startswith('part'): + self.partnum += 1 + self.partitions.append(Partition(parsed, self.partnum)) + elif line.startswith('include'): + self._parse(parser, parsed.path) + elif line.startswith('bootloader'): + if not self.bootloader: + self.bootloader = parsed + else: + err = "%s:%d: more than one bootloader specified" \ + % (confpath, lineno) + raise KickStartError(err) diff --git a/scripts/lib/wic/msger.py b/scripts/lib/wic/msger.py new file mode 100644 index 0000000..b737554 --- /dev/null +++ b/scripts/lib/wic/msger.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python -tt +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (c) 2009, 2010, 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import sys +import re +import time + +__ALL__ = ['set_mode', + 'get_loglevel', + 'set_loglevel', + 'set_logfile', + 'raw', + 'debug', + 'verbose', + 'info', + 'warning', + 'error', + 'ask', + 'pause', + ] + +# COLORs in ANSI +INFO_COLOR = 32 # green +WARN_COLOR = 33 # yellow +ERR_COLOR = 31 # red +ASK_COLOR = 34 # blue +NO_COLOR = 0 + +PREFIX_RE = re.compile('^<(.*?)>\s*(.*)', re.S) + +INTERACTIVE = True + +LOG_LEVEL = 1 +LOG_LEVELS = { + 'quiet': 0, + 'normal': 1, + 'verbose': 2, + 'debug': 3, + 'never': 4, +} + +LOG_FILE_FP = None +LOG_CONTENT = '' +CATCHERR_BUFFILE_FD = -1 +CATCHERR_BUFFILE_PATH = None +CATCHERR_SAVED_2 = -1 + +def _general_print(head, color, msg=None, stream=None, level='normal'): + global LOG_CONTENT + if not stream: + stream = sys.stdout + + if LOG_LEVELS[level] > LOG_LEVEL: + # skip + return + + # encode raw 'unicode' str to utf8 encoded str + if msg and isinstance(msg, unicode): + msg = msg.encode('utf-8', 'ignore') + + errormsg = '' + if CATCHERR_BUFFILE_FD > 0: + size = os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_END) + os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_SET) + errormsg = os.read(CATCHERR_BUFFILE_FD, size) + os.ftruncate(CATCHERR_BUFFILE_FD, 0) + + # append error msg to LOG + if errormsg: + LOG_CONTENT += errormsg + + # append normal msg to LOG + save_msg = msg.strip() if msg else None + if save_msg: + timestr = time.strftime("[%m/%d %H:%M:%S %Z] ", time.localtime()) + LOG_CONTENT += timestr + save_msg + '\n' + + if errormsg: + _color_print('', NO_COLOR, errormsg, stream, level) + + _color_print(head, color, msg, stream, level) + +def _color_print(head, color, msg, stream, level): + colored = True + if color == NO_COLOR or \ + not stream.isatty() or \ + os.getenv('ANSI_COLORS_DISABLED') is not None: + colored = False + + if head.startswith('\r'): + # need not \n at last + newline = False + else: + newline = True + + if colored: + head = '\033[%dm%s:\033[0m ' %(color, head) + if not newline: + # ESC cmd to clear line + head = '\033[2K' + head + else: + if head: + head += ': ' + if head.startswith('\r'): + head = head.lstrip() + newline = True + + if msg is not None: + if isinstance(msg, unicode): + msg = msg.encode('utf8', 'ignore') + + stream.write('%s%s' % (head, msg)) + if newline: + stream.write('\n') + + stream.flush() + +def _color_perror(head, color, msg, level='normal'): + if CATCHERR_BUFFILE_FD > 0: + _general_print(head, color, msg, sys.stdout, level) + else: + _general_print(head, color, msg, sys.stderr, level) + +def _split_msg(head, msg): + if isinstance(msg, list): + msg = '\n'.join(map(str, msg)) + + if msg.startswith('\n'): + # means print \n at first + msg = msg.lstrip() + head = '\n' + head + + elif msg.startswith('\r'): + # means print \r at first + msg = msg.lstrip() + head = '\r' + head + + match = PREFIX_RE.match(msg) + if match: + head += ' <%s>' % match.group(1) + msg = match.group(2) + + return head, msg + +def get_loglevel(): + return (k for k, v in LOG_LEVELS.items() if v == LOG_LEVEL).next() + +def set_loglevel(level): + global LOG_LEVEL + if level not in LOG_LEVELS: + # no effect + return + + LOG_LEVEL = LOG_LEVELS[level] + +def set_interactive(mode=True): + global INTERACTIVE + if mode: + INTERACTIVE = True + else: + INTERACTIVE = False + +def log(msg=''): + # log msg to LOG_CONTENT then save to logfile + global LOG_CONTENT + if msg: + LOG_CONTENT += msg + +def raw(msg=''): + _general_print('', NO_COLOR, msg) + +def info(msg): + head, msg = _split_msg('Info', msg) + _general_print(head, INFO_COLOR, msg) + +def verbose(msg): + head, msg = _split_msg('Verbose', msg) + _general_print(head, INFO_COLOR, msg, level='verbose') + +def warning(msg): + head, msg = _split_msg('Warning', msg) + _color_perror(head, WARN_COLOR, msg) + +def debug(msg): + head, msg = _split_msg('Debug', msg) + _color_perror(head, ERR_COLOR, msg, level='debug') + +def error(msg): + head, msg = _split_msg('Error', msg) + _color_perror(head, ERR_COLOR, msg) + sys.exit(1) + +def ask(msg, default=True): + _general_print('\rQ', ASK_COLOR, '') + try: + if default: + msg += '(Y/n) ' + else: + msg += '(y/N) ' + if INTERACTIVE: + while True: + repl = raw_input(msg) + if repl.lower() == 'y': + return True + elif repl.lower() == 'n': + return False + elif not repl.strip(): + # <Enter> + return default + + # else loop + else: + if default: + msg += ' Y' + else: + msg += ' N' + _general_print('', NO_COLOR, msg) + + return default + except KeyboardInterrupt: + sys.stdout.write('\n') + sys.exit(2) + +def choice(msg, choices, default=0): + if default >= len(choices): + return None + _general_print('\rQ', ASK_COLOR, '') + try: + msg += " [%s] " % '/'.join(choices) + if INTERACTIVE: + while True: + repl = raw_input(msg) + if repl in choices: + return repl + elif not repl.strip(): + return choices[default] + else: + msg += choices[default] + _general_print('', NO_COLOR, msg) + + return choices[default] + except KeyboardInterrupt: + sys.stdout.write('\n') + sys.exit(2) + +def pause(msg=None): + if INTERACTIVE: + _general_print('\rQ', ASK_COLOR, '') + if msg is None: + msg = 'press <ENTER> to continue ...' + raw_input(msg) + +def set_logfile(fpath): + global LOG_FILE_FP + + def _savelogf(): + if LOG_FILE_FP: + with open(LOG_FILE_FP, 'w') as log: + log.write(LOG_CONTENT) + + if LOG_FILE_FP is not None: + warning('duplicate log file configuration') + + LOG_FILE_FP = fpath + + import atexit + atexit.register(_savelogf) + +def enable_logstderr(fpath): + global CATCHERR_BUFFILE_FD + global CATCHERR_BUFFILE_PATH + global CATCHERR_SAVED_2 + + if os.path.exists(fpath): + os.remove(fpath) + CATCHERR_BUFFILE_PATH = fpath + CATCHERR_BUFFILE_FD = os.open(CATCHERR_BUFFILE_PATH, os.O_RDWR|os.O_CREAT) + CATCHERR_SAVED_2 = os.dup(2) + os.dup2(CATCHERR_BUFFILE_FD, 2) + +def disable_logstderr(): + global CATCHERR_BUFFILE_FD + global CATCHERR_BUFFILE_PATH + global CATCHERR_SAVED_2 + + raw(msg=None) # flush message buffer and print it. + os.dup2(CATCHERR_SAVED_2, 2) + os.close(CATCHERR_SAVED_2) + os.close(CATCHERR_BUFFILE_FD) + os.unlink(CATCHERR_BUFFILE_PATH) + CATCHERR_BUFFILE_FD = -1 + CATCHERR_BUFFILE_PATH = None + CATCHERR_SAVED_2 = -1 diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py new file mode 100644 index 0000000..f40d1bc --- /dev/null +++ b/scripts/lib/wic/partition.py @@ -0,0 +1,414 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2013-2016 Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This module provides the OpenEmbedded partition object definitions. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# Ed Bartosh <ed.bartosh> (at] linux.intel.com> + +import os +import tempfile +import uuid + +from wic.utils.oe.misc import msger, parse_sourceparams +from wic.utils.oe.misc import exec_cmd, exec_native_cmd +from wic.plugin import pluginmgr + +partition_methods = { + "do_stage_partition":None, + "do_prepare_partition":None, + "do_configure_partition":None, +} + +class Partition(object): + + def __init__(self, args, lineno): + self.args = args + self.active = args.active + self.align = args.align + self.disk = args.disk + self.extra_space = args.extra_space + self.fsopts = args.fsopts + self.fstype = args.fstype + self.label = args.label + self.mountpoint = args.mountpoint + self.no_table = args.no_table + self.overhead_factor = args.overhead_factor + self.part_type = args.part_type + self.rootfs_dir = args.rootfs_dir + self.size = args.size + self.source = args.source + self.sourceparams = args.sourceparams + self.use_uuid = args.use_uuid + self.uuid = args.uuid + if args.use_uuid and not self.uuid: + self.uuid = str(uuid.uuid4()) + + self.lineno = lineno + self.source_file = "" + self.sourceparams_dict = {} + + def get_extra_block_count(self, current_blocks): + """ + The --size param is reflected in self.size (in kB), and we already + have current_blocks (1k) blocks, calculate and return the + number of (1k) blocks we need to add to get to --size, 0 if + we're already there or beyond. + """ + msger.debug("Requested partition size for %s: %d" % \ + (self.mountpoint, self.size)) + + if not self.size: + return 0 + + requested_blocks = self.size + + msger.debug("Requested blocks %d, current_blocks %d" % \ + (requested_blocks, current_blocks)) + + if requested_blocks > current_blocks: + return requested_blocks - current_blocks + else: + return 0 + + def prepare(self, creator, cr_workdir, oe_builddir, rootfs_dir, + bootimg_dir, kernel_dir, native_sysroot): + """ + Prepare content for individual partitions, depending on + partition command parameters. + """ + if self.sourceparams: + self.sourceparams_dict = parse_sourceparams(self.sourceparams) + + if not self.source: + if not self.size: + msger.error("The %s partition has a size of zero. Please " + "specify a non-zero --size for that partition." % \ + self.mountpoint) + if self.fstype and self.fstype == "swap": + self.prepare_swap_partition(cr_workdir, oe_builddir, + native_sysroot) + self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype) + elif self.fstype: + rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label, + self.lineno, self.fstype) + if os.path.isfile(rootfs): + os.remove(rootfs) + for prefix in ("ext", "btrfs", "vfat", "squashfs"): + if self.fstype.startswith(prefix): + method = getattr(self, + "prepare_empty_partition_" + prefix) + method(rootfs, oe_builddir, native_sysroot) + self.source_file = rootfs + break + return + + plugins = pluginmgr.get_source_plugins() + + if self.source not in plugins: + msger.error("The '%s' --source specified for %s doesn't exist.\n\t" + "See 'wic list source-plugins' for a list of available" + " --sources.\n\tSee 'wic help source-plugins' for " + "details on adding a new source plugin." % \ + (self.source, self.mountpoint)) + + self._source_methods = pluginmgr.get_source_plugin_methods(\ + self.source, partition_methods) + self._source_methods["do_configure_partition"](self, self.sourceparams_dict, + creator, cr_workdir, + oe_builddir, + bootimg_dir, + kernel_dir, + native_sysroot) + self._source_methods["do_stage_partition"](self, self.sourceparams_dict, + creator, cr_workdir, + oe_builddir, + bootimg_dir, kernel_dir, + native_sysroot) + self._source_methods["do_prepare_partition"](self, self.sourceparams_dict, + creator, cr_workdir, + oe_builddir, + bootimg_dir, kernel_dir, rootfs_dir, + native_sysroot) + + def prepare_rootfs_from_fs_image(self, cr_workdir, oe_builddir, + rootfs_dir): + """ + Handle an already-created partition e.g. xxx.ext3 + """ + rootfs = oe_builddir + du_cmd = "du -Lbks %s" % rootfs + out = exec_cmd(du_cmd) + rootfs_size = out.split()[0] + + self.size = rootfs_size + self.source_file = rootfs + + def prepare_rootfs(self, cr_workdir, oe_builddir, rootfs_dir, + native_sysroot): + """ + Prepare content for a rootfs partition i.e. create a partition + and fill it from a /rootfs dir. + + Currently handles ext2/3/4, btrfs and vfat. + """ + p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot) + p_localstatedir = os.environ.get("PSEUDO_LOCALSTATEDIR", + "%s/../pseudo" % rootfs_dir) + p_passwd = os.environ.get("PSEUDO_PASSWD", rootfs_dir) + p_nosymlinkexp = os.environ.get("PSEUDO_NOSYMLINKEXP", "1") + pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix + pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % p_localstatedir + pseudo += "export PSEUDO_PASSWD=%s;" % p_passwd + pseudo += "export PSEUDO_NOSYMLINKEXP=%s;" % p_nosymlinkexp + pseudo += "%s/usr/bin/pseudo " % native_sysroot + + rootfs = "%s/rootfs_%s.%s.%s" % (cr_workdir, self.label, + self.lineno, self.fstype) + if os.path.isfile(rootfs): + os.remove(rootfs) + + for prefix in ("ext", "btrfs", "vfat", "squashfs"): + if self.fstype.startswith(prefix): + method = getattr(self, "prepare_rootfs_" + prefix) + method(rootfs, oe_builddir, rootfs_dir, native_sysroot, pseudo) + + self.source_file = rootfs + + # get the rootfs size in the right units for kickstart (kB) + du_cmd = "du -Lbks %s" % rootfs + out = exec_cmd(du_cmd) + self.size = out.split()[0] + + break + + def prepare_rootfs_ext(self, rootfs, oe_builddir, rootfs_dir, + native_sysroot, pseudo): + """ + Prepare content for an ext2/3/4 rootfs partition. + """ + du_cmd = "du -ks %s" % rootfs_dir + out = exec_cmd(du_cmd) + actual_rootfs_size = int(out.split()[0]) + + extra_blocks = self.get_extra_block_count(actual_rootfs_size) + if extra_blocks < self.extra_space: + extra_blocks = self.extra_space + + rootfs_size = actual_rootfs_size + extra_blocks + rootfs_size *= self.overhead_factor + + msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \ + (extra_blocks, self.mountpoint, rootfs_size)) + + dd_cmd = "dd if=/dev/zero of=%s bs=1024 seek=%d count=0 bs=1k" % \ + (rootfs, rootfs_size) + exec_cmd(dd_cmd) + + extra_imagecmd = "-i 8192" + + label_str = "" + if self.label: + label_str = "-L %s" % self.label + + mkfs_cmd = "mkfs.%s -F %s %s %s -d %s" % \ + (self.fstype, extra_imagecmd, rootfs, label_str, rootfs_dir) + exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) + + def prepare_rootfs_btrfs(self, rootfs, oe_builddir, rootfs_dir, + native_sysroot, pseudo): + """ + Prepare content for a btrfs rootfs partition. + + Currently handles ext2/3/4 and btrfs. + """ + du_cmd = "du -ks %s" % rootfs_dir + out = exec_cmd(du_cmd) + actual_rootfs_size = int(out.split()[0]) + + extra_blocks = self.get_extra_block_count(actual_rootfs_size) + if extra_blocks < self.extra_space: + extra_blocks = self.extra_space + + rootfs_size = actual_rootfs_size + extra_blocks + rootfs_size *= self.overhead_factor + + msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \ + (extra_blocks, self.mountpoint, rootfs_size)) + + dd_cmd = "dd if=/dev/zero of=%s bs=1024 seek=%d count=0 bs=1k" % \ + (rootfs, rootfs_size) + exec_cmd(dd_cmd) + + label_str = "" + if self.label: + label_str = "-L %s" % self.label + + mkfs_cmd = "mkfs.%s -b %d -r %s %s %s" % \ + (self.fstype, rootfs_size * 1024, rootfs_dir, label_str, rootfs) + exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) + + def prepare_rootfs_vfat(self, rootfs, oe_builddir, rootfs_dir, + native_sysroot, pseudo): + """ + Prepare content for a vfat rootfs partition. + """ + du_cmd = "du -bks %s" % rootfs_dir + out = exec_cmd(du_cmd) + blocks = int(out.split()[0]) + + extra_blocks = self.get_extra_block_count(blocks) + if extra_blocks < self.extra_space: + extra_blocks = self.extra_space + + blocks += extra_blocks + + msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \ + (extra_blocks, self.mountpoint, blocks)) + + # Ensure total sectors is an integral number of sectors per + # track or mcopy will complain. Sectors are 512 bytes, and we + # generate images with 32 sectors per track. This calculation + # is done in blocks, thus the mod by 16 instead of 32. Apply + # sector count fix only when needed. + if blocks % 16 != 0: + blocks += (16 - (blocks % 16)) + + label_str = "-n boot" + if self.label: + label_str = "-n %s" % self.label + + dosfs_cmd = "mkdosfs %s -S 512 -C %s %d" % (label_str, rootfs, blocks) + exec_native_cmd(dosfs_cmd, native_sysroot) + + mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir) + exec_native_cmd(mcopy_cmd, native_sysroot) + + chmod_cmd = "chmod 644 %s" % rootfs + exec_cmd(chmod_cmd) + + def prepare_rootfs_squashfs(self, rootfs, oe_builddir, rootfs_dir, + native_sysroot, pseudo): + """ + Prepare content for a squashfs rootfs partition. + """ + squashfs_cmd = "mksquashfs %s %s -noappend" % \ + (rootfs_dir, rootfs) + exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo) + + def prepare_empty_partition_ext(self, rootfs, oe_builddir, + native_sysroot): + """ + Prepare an empty ext2/3/4 partition. + """ + dd_cmd = "dd if=/dev/zero of=%s bs=1k seek=%d count=0" % \ + (rootfs, self.size) + exec_cmd(dd_cmd) + + extra_imagecmd = "-i 8192" + + label_str = "" + if self.label: + label_str = "-L %s" % self.label + + mkfs_cmd = "mkfs.%s -F %s %s %s" % \ + (self.fstype, extra_imagecmd, label_str, rootfs) + exec_native_cmd(mkfs_cmd, native_sysroot) + + def prepare_empty_partition_btrfs(self, rootfs, oe_builddir, + native_sysroot): + """ + Prepare an empty btrfs partition. + """ + dd_cmd = "dd if=/dev/zero of=%s bs=1k seek=%d count=0" % \ + (rootfs, self.size) + exec_cmd(dd_cmd) + + label_str = "" + if self.label: + label_str = "-L %s" % self.label + + mkfs_cmd = "mkfs.%s -b %d %s %s" % \ + (self.fstype, self.size * 1024, label_str, rootfs) + exec_native_cmd(mkfs_cmd, native_sysroot) + + def prepare_empty_partition_vfat(self, rootfs, oe_builddir, + native_sysroot): + """ + Prepare an empty vfat partition. + """ + blocks = self.size + + label_str = "-n boot" + if self.label: + label_str = "-n %s" % self.label + + dosfs_cmd = "mkdosfs %s -S 512 -C %s %d" % (label_str, rootfs, blocks) + exec_native_cmd(dosfs_cmd, native_sysroot) + + chmod_cmd = "chmod 644 %s" % rootfs + exec_cmd(chmod_cmd) + + def prepare_empty_partition_squashfs(self, cr_workdir, oe_builddir, + native_sysroot): + """ + Prepare an empty squashfs partition. + """ + msger.warning("Creating of an empty squashfs %s partition was attempted. " \ + "Proceeding as requested." % self.mountpoint) + + path = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype) + os.path.isfile(path) and os.remove(path) + + # it is not possible to create a squashfs without source data, + # thus prepare an empty temp dir that is used as source + tmpdir = tempfile.mkdtemp() + + squashfs_cmd = "mksquashfs %s %s -noappend" % \ + (tmpdir, path) + exec_native_cmd(squashfs_cmd, native_sysroot) + + os.rmdir(tmpdir) + + # get the rootfs size in the right units for kickstart (kB) + du_cmd = "du -Lbks %s" % path + out = exec_cmd(du_cmd) + fs_size = out.split()[0] + + self.size = fs_size + + def prepare_swap_partition(self, cr_workdir, oe_builddir, native_sysroot): + """ + Prepare a swap partition. + """ + path = "%s/fs.%s" % (cr_workdir, self.fstype) + + dd_cmd = "dd if=/dev/zero of=%s bs=1k seek=%d count=0" % \ + (path, self.size) + exec_cmd(dd_cmd) + + import uuid + label_str = "" + if self.label: + label_str = "-L %s" % self.label + mkswap_cmd = "mkswap %s -U %s %s" % (label_str, str(uuid.uuid1()), path) + exec_native_cmd(mkswap_cmd, native_sysroot) + diff --git a/scripts/lib/wic/plugin.py b/scripts/lib/wic/plugin.py new file mode 100644 index 0000000..151ff31 --- /dev/null +++ b/scripts/lib/wic/plugin.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os, sys + +from wic import msger +from wic import pluginbase +from wic.utils import errors +from wic.utils.oe.misc import get_bitbake_var + +__ALL__ = ['PluginMgr', 'pluginmgr'] + +PLUGIN_TYPES = ["imager", "source"] + +PLUGIN_DIR = "/lib/wic/plugins" # relative to scripts +SCRIPTS_PLUGIN_DIR = "scripts" + PLUGIN_DIR + +class PluginMgr(object): + plugin_dirs = {} + + # make the manager class as singleton + _instance = None + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(PluginMgr, cls).__new__(cls, *args, **kwargs) + + return cls._instance + + def __init__(self): + wic_path = os.path.dirname(__file__) + eos = wic_path.rfind('scripts') + len('scripts') + scripts_path = wic_path[:eos] + self.scripts_path = scripts_path + self.plugin_dir = scripts_path + PLUGIN_DIR + self.layers_path = None + + def _build_plugin_dir_list(self, plugin_dir, ptype): + if self.layers_path is None: + self.layers_path = get_bitbake_var("BBLAYERS") + layer_dirs = [] + + if self.layers_path is not None: + for layer_path in self.layers_path.split(): + path = os.path.join(layer_path, SCRIPTS_PLUGIN_DIR, ptype) + layer_dirs.append(path) + + path = os.path.join(plugin_dir, ptype) + layer_dirs.append(path) + + return layer_dirs + + def append_dirs(self, dirs): + for path in dirs: + self._add_plugindir(path) + + # load all the plugins AGAIN + self._load_all() + + def _add_plugindir(self, path): + path = os.path.abspath(os.path.expanduser(path)) + + if not os.path.isdir(path): + return + + if path not in self.plugin_dirs: + self.plugin_dirs[path] = False + # the value True/False means "loaded" + + def _load_all(self): + for (pdir, loaded) in self.plugin_dirs.iteritems(): + if loaded: + continue + + sys.path.insert(0, pdir) + for mod in [x[:-3] for x in os.listdir(pdir) if x.endswith(".py")]: + if mod and mod != '__init__': + if mod in sys.modules: + #self.plugin_dirs[pdir] = True + msger.warning("Module %s already exists, skip" % mod) + else: + try: + pymod = __import__(mod) + self.plugin_dirs[pdir] = True + msger.debug("Plugin module %s:%s imported"\ + % (mod, pymod.__file__)) + except ImportError, err: + msg = 'Failed to load plugin %s/%s: %s' \ + % (os.path.basename(pdir), mod, err) + msger.warning(msg) + + del sys.path[0] + + def get_plugins(self, ptype): + """ the return value is dict of name:class pairs """ + + if ptype not in PLUGIN_TYPES: + raise errors.CreatorError('%s is not valid plugin type' % ptype) + + plugins_dir = self._build_plugin_dir_list(self.plugin_dir, ptype) + + self.append_dirs(plugins_dir) + + return pluginbase.get_plugins(ptype) + + def get_source_plugins(self): + """ + Return list of available source plugins. + """ + plugins_dir = self._build_plugin_dir_list(self.plugin_dir, 'source') + + self.append_dirs(plugins_dir) + + return self.get_plugins('source') + + + def get_source_plugin_methods(self, source_name, methods): + """ + The methods param is a dict with the method names to find. On + return, the dict values will be filled in with pointers to the + corresponding methods. If one or more methods are not found, + None is returned. + """ + return_methods = None + for _source_name, klass in self.get_plugins('source').iteritems(): + if _source_name == source_name: + for _method_name in methods.keys(): + if not hasattr(klass, _method_name): + msger.warning("Unimplemented %s source interface for: %s"\ + % (_method_name, _source_name)) + return None + func = getattr(klass, _method_name) + methods[_method_name] = func + return_methods = methods + return return_methods + +pluginmgr = PluginMgr() diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py new file mode 100644 index 0000000..ee8fe95 --- /dev/null +++ b/scripts/lib/wic/pluginbase.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +from wic import msger + +class _Plugin(object): + class __metaclass__(type): + def __init__(cls, name, bases, attrs): + if not hasattr(cls, 'plugins'): + cls.plugins = {} + + elif 'wic_plugin_type' in attrs: + if attrs['wic_plugin_type'] not in cls.plugins: + cls.plugins[attrs['wic_plugin_type']] = {} + + elif hasattr(cls, 'wic_plugin_type') and 'name' in attrs: + cls.plugins[cls.wic_plugin_type][attrs['name']] = cls + + def show_plugins(cls): + for cls in cls.plugins[cls.wic_plugin_type]: + print cls + + def get_plugins(cls): + return cls.plugins + + +class ImagerPlugin(_Plugin): + wic_plugin_type = "imager" + + +class SourcePlugin(_Plugin): + wic_plugin_type = "source" + """ + The methods that can be implemented by --source plugins. + + Any methods not implemented in a subclass inherit these. + """ + + @classmethod + def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir, + bootimg_dir, kernel_dir, native_sysroot): + """ + Called after all partitions have been prepared and assembled into a + disk image. This provides a hook to allow finalization of a + disk image e.g. to write an MBR to it. + """ + msger.debug("SourcePlugin: do_install_disk: disk: %s" % disk_name) + + @classmethod + def do_stage_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + native_sysroot): + """ + Special content staging hook called before do_prepare_partition(), + normally empty. + + Typically, a partition will just use the passed-in parame e.g + straight bootimg_dir, etc, but in some cases, things need to + be more tailored e.g. to use a deploy dir + /boot, etc. This + hook allows those files to be staged in a customized fashion. + Not that get_bitbake_var() allows you to acces non-standard + variables that you might want to use for this. + """ + msger.debug("SourcePlugin: do_stage_partition: part: %s" % part) + + @classmethod + def do_configure_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + native_sysroot): + """ + Called before do_prepare_partition(), typically used to create + custom configuration files for a partition, for example + syslinux or grub config files. + """ + msger.debug("SourcePlugin: do_configure_partition: part: %s" % part) + + @classmethod + def do_prepare_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, rootfs_dir, + native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + """ + msger.debug("SourcePlugin: do_prepare_partition: part: %s" % part) + +def get_plugins(typen): + plugins = ImagerPlugin.get_plugins() + if typen in plugins: + return plugins[typen] + else: + return None + +__all__ = ['ImagerPlugin', 'SourcePlugin', 'get_plugins'] diff --git a/scripts/lib/wic/plugins/imager/direct_plugin.py b/scripts/lib/wic/plugins/imager/direct_plugin.py new file mode 100644 index 0000000..6d3f46c --- /dev/null +++ b/scripts/lib/wic/plugins/imager/direct_plugin.py @@ -0,0 +1,102 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This implements the 'direct' imager plugin class for 'wic' +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# + +from wic.utils import errors +from wic.conf import configmgr + +import wic.imager.direct as direct +from wic.pluginbase import ImagerPlugin + +class DirectPlugin(ImagerPlugin): + """ + Install a system into a file containing a partitioned disk image. + + An image file is formatted with a partition table, each partition + created from a rootfs or other OpenEmbedded build artifact and dd'ed + into the virtual disk. The disk image can subsequently be dd'ed onto + media and used on actual hardware. + """ + + name = 'direct' + + @classmethod + def __rootfs_dir_to_dict(cls, rootfs_dirs): + """ + Gets a string that contain 'connection=dir' splitted by + space and return a dict + """ + krootfs_dir = {} + for rootfs_dir in rootfs_dirs.split(' '): + key, val = rootfs_dir.split('=') + krootfs_dir[key] = val + + return krootfs_dir + + @classmethod + def do_create(cls, opts, *args): + """ + Create direct image, called from creator as 'direct' cmd + """ + if len(args) != 8: + raise errors.Usage("Extra arguments given") + + native_sysroot = args[0] + kernel_dir = args[1] + bootimg_dir = args[2] + rootfs_dir = args[3] + + creatoropts = configmgr.create + ksconf = args[4] + + image_output_dir = args[5] + oe_builddir = args[6] + compressor = args[7] + + krootfs_dir = cls.__rootfs_dir_to_dict(rootfs_dir) + + configmgr._ksconf = ksconf + + creator = direct.DirectImageCreator(oe_builddir, + image_output_dir, + krootfs_dir, + bootimg_dir, + kernel_dir, + native_sysroot, + compressor, + creatoropts) + + try: + creator.create() + creator.assemble() + creator.finalize() + creator.print_outimage_info() + + except errors.CreatorError: + raise + finally: + creator.cleanup() + + return 0 diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg-efi.py new file mode 100644 index 0000000..a4734c9 --- /dev/null +++ b/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -0,0 +1,237 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2014, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This implements the 'bootimg-efi' source plugin class for 'wic' +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# + +import os +import shutil + +from wic import msger +from wic.pluginbase import SourcePlugin +from wic.utils.misc import get_custom_config +from wic.utils.oe.misc import exec_cmd, exec_native_cmd, get_bitbake_var, \ + BOOTDD_EXTRA_SPACE + +class BootimgEFIPlugin(SourcePlugin): + """ + Create EFI boot partition. + This plugin supports GRUB 2 and gummiboot bootloaders. + """ + + name = 'bootimg-efi' + + @classmethod + def do_configure_grubefi(cls, hdddir, creator, cr_workdir): + """ + Create loader-specific (grub-efi) config + """ + configfile = creator.ks.bootloader.configfile + custom_cfg = None + if configfile: + custom_cfg = get_custom_config(configfile) + if custom_cfg: + # Use a custom configuration for grub + grubefi_conf = custom_cfg + msger.debug("Using custom configuration file " + "%s for grub.cfg" % configfile) + else: + msger.error("configfile is specified but failed to " + "get it from %s." % configfile) + + if not custom_cfg: + # Create grub configuration using parameters from wks file + bootloader = creator.ks.bootloader + + grubefi_conf = "" + grubefi_conf += "serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1\n" + grubefi_conf += "default=boot\n" + grubefi_conf += "timeout=%s\n" % bootloader.timeout + grubefi_conf += "menuentry 'boot'{\n" + + kernel = "/bzImage" + + grubefi_conf += "linux %s root=%s rootwait %s\n" \ + % (kernel, creator.rootdev, bootloader.append) + grubefi_conf += "}\n" + + msger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg" \ + % cr_workdir) + cfg = open("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir, "w") + cfg.write(grubefi_conf) + cfg.close() + + @classmethod + def do_configure_gummiboot(cls, hdddir, creator, cr_workdir): + """ + Create loader-specific (gummiboot) config + """ + install_cmd = "install -d %s/loader" % hdddir + exec_cmd(install_cmd) + + install_cmd = "install -d %s/loader/entries" % hdddir + exec_cmd(install_cmd) + + bootloader = creator.ks.bootloader + + loader_conf = "" + loader_conf += "default boot\n" + loader_conf += "timeout %d\n" % bootloader.timeout + + msger.debug("Writing gummiboot config %s/hdd/boot/loader/loader.conf" \ + % cr_workdir) + cfg = open("%s/hdd/boot/loader/loader.conf" % cr_workdir, "w") + cfg.write(loader_conf) + cfg.close() + + configfile = creator.ks.bootloader.configfile + custom_cfg = None + if configfile: + custom_cfg = get_custom_config(configfile) + if custom_cfg: + # Use a custom configuration for gummiboot + boot_conf = custom_cfg + msger.debug("Using custom configuration file " + "%s for gummiboots's boot.conf" % configfile) + else: + msger.error("configfile is specified but failed to " + "get it from %s." % configfile) + + if not custom_cfg: + # Create gummiboot configuration using parameters from wks file + kernel = "/bzImage" + + boot_conf = "" + boot_conf += "title boot\n" + boot_conf += "linux %s\n" % kernel + boot_conf += "options LABEL=Boot root=%s %s\n" % \ + (creator.rootdev, bootloader.append) + + msger.debug("Writing gummiboot config %s/hdd/boot/loader/entries/boot.conf" \ + % cr_workdir) + cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") + cfg.write(boot_conf) + cfg.close() + + + @classmethod + def do_configure_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + native_sysroot): + """ + Called before do_prepare_partition(), creates loader-specific config + """ + hdddir = "%s/hdd/boot" % cr_workdir + + install_cmd = "install -d %s/EFI/BOOT" % hdddir + exec_cmd(install_cmd) + + try: + if source_params['loader'] == 'grub-efi': + cls.do_configure_grubefi(hdddir, creator, cr_workdir) + elif source_params['loader'] == 'gummiboot': + cls.do_configure_gummiboot(hdddir, creator, cr_workdir) + else: + msger.error("unrecognized bootimg-efi loader: %s" % source_params['loader']) + except KeyError: + msger.error("bootimg-efi requires a loader, none specified") + + + @classmethod + def do_prepare_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + rootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + In this case, prepare content for an EFI (grub) boot partition. + """ + if not bootimg_dir: + bootimg_dir = get_bitbake_var("HDDDIR") + if not bootimg_dir: + msger.error("Couldn't find HDDDIR, exiting\n") + # just so the result notes display it + creator.set_bootimg_dir(bootimg_dir) + + staging_kernel_dir = kernel_dir + + hdddir = "%s/hdd/boot" % cr_workdir + + install_cmd = "install -m 0644 %s/bzImage %s/bzImage" % \ + (staging_kernel_dir, hdddir) + exec_cmd(install_cmd) + + try: + if source_params['loader'] == 'grub-efi': + shutil.copyfile("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir, + "%s/grub.cfg" % cr_workdir) + cp_cmd = "cp %s/EFI/BOOT/* %s/EFI/BOOT" % (bootimg_dir, hdddir) + exec_cmd(cp_cmd, True) + shutil.move("%s/grub.cfg" % cr_workdir, + "%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir) + elif source_params['loader'] == 'gummiboot': + cp_cmd = "cp %s/EFI/BOOT/* %s/EFI/BOOT" % (bootimg_dir, hdddir) + exec_cmd(cp_cmd, True) + else: + msger.error("unrecognized bootimg-efi loader: %s" % source_params['loader']) + except KeyError: + msger.error("bootimg-efi requires a loader, none specified") + + du_cmd = "du -bks %s" % hdddir + out = exec_cmd(du_cmd) + blocks = int(out.split()[0]) + + extra_blocks = part.get_extra_block_count(blocks) + + if extra_blocks < BOOTDD_EXTRA_SPACE: + extra_blocks = BOOTDD_EXTRA_SPACE + + blocks += extra_blocks + + msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \ + (extra_blocks, part.mountpoint, blocks)) + + # Ensure total sectors is an integral number of sectors per + # track or mcopy will complain. Sectors are 512 bytes, and we + # generate images with 32 sectors per track. This calculation is + # done in blocks, thus the mod by 16 instead of 32. + blocks += (16 - (blocks % 16)) + + # dosfs image, created by mkdosfs + bootimg = "%s/boot.img" % cr_workdir + + dosfs_cmd = "mkdosfs -n efi -C %s %d" % (bootimg, blocks) + exec_native_cmd(dosfs_cmd, native_sysroot) + + mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir) + exec_native_cmd(mcopy_cmd, native_sysroot) + + chmod_cmd = "chmod 644 %s" % bootimg + exec_cmd(chmod_cmd) + + du_cmd = "du -Lbks %s" % bootimg + out = exec_cmd(du_cmd) + bootimg_size = out.split()[0] + + part.size = bootimg_size + part.source_file = bootimg diff --git a/scripts/lib/wic/plugins/source/bootimg-partition.py b/scripts/lib/wic/plugins/source/bootimg-partition.py new file mode 100644 index 0000000..b76c121 --- /dev/null +++ b/scripts/lib/wic/plugins/source/bootimg-partition.py @@ -0,0 +1,140 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This implements the 'bootimg-partition' source plugin class for +# 'wic'. The plugin creates an image of boot partition, copying over +# files listed in IMAGE_BOOT_FILES bitbake variable. +# +# AUTHORS +# Maciej Borzecki <maciej.borzecki (at] open-rnd.pl> +# + +import os +import re + +from wic import msger +from wic.pluginbase import SourcePlugin +from wic.utils.oe.misc import exec_cmd, get_bitbake_var +from glob import glob + +class BootimgPartitionPlugin(SourcePlugin): + """ + Create an image of boot partition, copying over files + listed in IMAGE_BOOT_FILES bitbake variable. + """ + + name = 'bootimg-partition' + + @classmethod + def do_install_disk(cls, disk, disk_name, cr, workdir, oe_builddir, + bootimg_dir, kernel_dir, native_sysroot): + """ + Called after all partitions have been prepared and assembled into a + disk image. Do nothing. + """ + pass + + @classmethod + def do_configure_partition(cls, part, source_params, cr, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + native_sysroot): + """ + Called before do_prepare_partition(). Possibly prepare + configuration files of some sort. + + """ + pass + + @classmethod + def do_prepare_partition(cls, part, source_params, cr, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + rootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + In this case, does the following: + - sets up a vfat partition + - copies all files listed in IMAGE_BOOT_FILES variable + """ + hdddir = "%s/boot" % cr_workdir + install_cmd = "install -d %s" % hdddir + exec_cmd(install_cmd) + + if not bootimg_dir: + bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not bootimg_dir: + msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting\n") + + msger.debug('Bootimg dir: %s' % bootimg_dir) + + boot_files = get_bitbake_var("IMAGE_BOOT_FILES") + + if not boot_files: + msger.error('No boot files defined, IMAGE_BOOT_FILES unset') + + msger.debug('Boot files: %s' % boot_files) + + # list of tuples (src_name, dst_name) + deploy_files = [] + for src_entry in re.findall(r'[\w;\-\./\*]+', boot_files): + if ';' in src_entry: + dst_entry = tuple(src_entry.split(';')) + if not dst_entry[0] or not dst_entry[1]: + msger.error('Malformed boot file entry: %s' % (src_entry)) + else: + dst_entry = (src_entry, src_entry) + + msger.debug('Destination entry: %r' % (dst_entry,)) + deploy_files.append(dst_entry) + + for deploy_entry in deploy_files: + src, dst = deploy_entry + install_task = [] + if '*' in src: + # by default install files under their basename + entry_name_fn = os.path.basename + if dst != src: + # unless a target name was given, then treat name + # as a directory and append a basename + entry_name_fn = lambda name: \ + os.path.join(dst, + os.path.basename(name)) + + srcs = glob(os.path.join(bootimg_dir, src)) + + msger.debug('Globbed sources: %s' % (', '.join(srcs))) + for entry in srcs: + entry_dst_name = entry_name_fn(entry) + install_task.append((entry, + os.path.join(hdddir, + entry_dst_name))) + else: + install_task = [(os.path.join(bootimg_dir, src), + os.path.join(hdddir, dst))] + + for task in install_task: + src_path, dst_path = task + msger.debug('Install %s as %s' % (os.path.basename(src_path), + dst_path)) + install_cmd = "install -m 0644 -D %s %s" \ + % (src_path, dst_path) + exec_cmd(install_cmd) + + msger.debug('Prepare boot partition using rootfs in %s' % (hdddir)) + part.prepare_rootfs(cr_workdir, oe_builddir, hdddir, + native_sysroot) + diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/scripts/lib/wic/plugins/source/bootimg-pcbios.py new file mode 100644 index 0000000..5b719bf --- /dev/null +++ b/scripts/lib/wic/plugins/source/bootimg-pcbios.py @@ -0,0 +1,210 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2014, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This implements the 'bootimg-pcbios' source plugin class for 'wic' +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# + +import os + +from wic.utils.errors import ImageError +from wic import msger +from wic.utils import runner +from wic.utils.misc import get_custom_config +from wic.pluginbase import SourcePlugin +from wic.utils.oe.misc import exec_cmd, exec_native_cmd, \ + get_bitbake_var, BOOTDD_EXTRA_SPACE + +class BootimgPcbiosPlugin(SourcePlugin): + """ + Create MBR boot partition and install syslinux on it. + """ + + name = 'bootimg-pcbios' + + @classmethod + def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir, + bootimg_dir, kernel_dir, native_sysroot): + """ + Called after all partitions have been prepared and assembled into a + disk image. In this case, we install the MBR. + """ + mbrfile = "%s/syslinux/" % bootimg_dir + if creator.ptable_format == 'msdos': + mbrfile += "mbr.bin" + elif creator.ptable_format == 'gpt': + mbrfile += "gptmbr.bin" + else: + msger.error("Unsupported partition table: %s" % creator.ptable_format) + + if not os.path.exists(mbrfile): + msger.error("Couldn't find %s. If using the -e option, do you " + "have the right MACHINE set in local.conf? If not, " + "is the bootimg_dir path correct?" % mbrfile) + + full_path = creator._full_path(workdir, disk_name, "direct") + msger.debug("Installing MBR on disk %s as %s with size %s bytes" \ + % (disk_name, full_path, disk['min_size'])) + + rcode = runner.show(['dd', 'if=%s' % mbrfile, + 'of=%s' % full_path, 'conv=notrunc']) + if rcode != 0: + raise ImageError("Unable to set MBR to %s" % full_path) + + @classmethod + def do_configure_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + native_sysroot): + """ + Called before do_prepare_partition(), creates syslinux config + """ + hdddir = "%s/hdd/boot" % cr_workdir + + install_cmd = "install -d %s" % hdddir + exec_cmd(install_cmd) + + bootloader = creator.ks.bootloader + + custom_cfg = None + if bootloader.configfile: + custom_cfg = get_custom_config(bootloader.configfile) + if custom_cfg: + # Use a custom configuration for grub + syslinux_conf = custom_cfg + msger.debug("Using custom configuration file " + "%s for syslinux.cfg" % bootloader.configfile) + else: + msger.error("configfile is specified but failed to " + "get it from %s." % bootloader.configfile) + + if not custom_cfg: + # Create syslinux configuration using parameters from wks file + splash = os.path.join(cr_workdir, "/hdd/boot/splash.jpg") + if os.path.exists(splash): + splashline = "menu background splash.jpg" + else: + splashline = "" + + syslinux_conf = "" + syslinux_conf += "PROMPT 0\n" + syslinux_conf += "TIMEOUT " + str(bootloader.timeout) + "\n" + syslinux_conf += "\n" + syslinux_conf += "ALLOWOPTIONS 1\n" + syslinux_conf += "SERIAL 0 115200\n" + syslinux_conf += "\n" + if splashline: + syslinux_conf += "%s\n" % splashline + syslinux_conf += "DEFAULT boot\n" + syslinux_conf += "LABEL boot\n" + + kernel = "/vmlinuz" + syslinux_conf += "KERNEL " + kernel + "\n" + + syslinux_conf += "APPEND label=boot root=%s %s\n" % \ + (creator.rootdev, bootloader.append) + + msger.debug("Writing syslinux config %s/hdd/boot/syslinux.cfg" \ + % cr_workdir) + cfg = open("%s/hdd/boot/syslinux.cfg" % cr_workdir, "w") + cfg.write(syslinux_conf) + cfg.close() + + @classmethod + def do_prepare_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + rootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + In this case, prepare content for legacy bios boot partition. + """ + def _has_syslinux(dirname): + if dirname: + syslinux = "%s/syslinux" % dirname + if os.path.exists(syslinux): + return True + return False + + if not _has_syslinux(bootimg_dir): + bootimg_dir = get_bitbake_var("STAGING_DATADIR") + if not bootimg_dir: + msger.error("Couldn't find STAGING_DATADIR, exiting\n") + if not _has_syslinux(bootimg_dir): + msger.error("Please build syslinux first\n") + # just so the result notes display it + creator.set_bootimg_dir(bootimg_dir) + + staging_kernel_dir = kernel_dir + + hdddir = "%s/hdd/boot" % cr_workdir + + install_cmd = "install -m 0644 %s/bzImage %s/vmlinuz" \ + % (staging_kernel_dir, hdddir) + exec_cmd(install_cmd) + + install_cmd = "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" \ + % (bootimg_dir, hdddir) + exec_cmd(install_cmd) + + du_cmd = "du -bks %s" % hdddir + out = exec_cmd(du_cmd) + blocks = int(out.split()[0]) + + extra_blocks = part.get_extra_block_count(blocks) + + if extra_blocks < BOOTDD_EXTRA_SPACE: + extra_blocks = BOOTDD_EXTRA_SPACE + + blocks += extra_blocks + + msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \ + (extra_blocks, part.mountpoint, blocks)) + + # Ensure total sectors is an integral number of sectors per + # track or mcopy will complain. Sectors are 512 bytes, and we + # generate images with 32 sectors per track. This calculation is + # done in blocks, thus the mod by 16 instead of 32. + blocks += (16 - (blocks % 16)) + + # dosfs image, created by mkdosfs + bootimg = "%s/boot.img" % cr_workdir + + dosfs_cmd = "mkdosfs -n boot -S 512 -C %s %d" % (bootimg, blocks) + exec_native_cmd(dosfs_cmd, native_sysroot) + + mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir) + exec_native_cmd(mcopy_cmd, native_sysroot) + + syslinux_cmd = "syslinux %s" % bootimg + exec_native_cmd(syslinux_cmd, native_sysroot) + + chmod_cmd = "chmod 644 %s" % bootimg + exec_cmd(chmod_cmd) + + du_cmd = "du -Lbks %s" % bootimg + out = exec_cmd(du_cmd) + bootimg_size = out.split()[0] + + part.size = int(out.split()[0]) + part.source_file = bootimg + + diff --git a/scripts/lib/wic/plugins/source/fsimage.py b/scripts/lib/wic/plugins/source/fsimage.py new file mode 100644 index 0000000..f894e89 --- /dev/null +++ b/scripts/lib/wic/plugins/source/fsimage.py @@ -0,0 +1,73 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import os + +from wic import msger +from wic.pluginbase import SourcePlugin +from wic.utils.oe.misc import get_bitbake_var + +class FSImagePlugin(SourcePlugin): + """ + Add an already existing filesystem image to the partition layout. + """ + + name = 'fsimage' + + @classmethod + def do_install_disk(cls, disk, disk_name, cr, workdir, oe_builddir, + bootimg_dir, kernel_dir, native_sysroot): + """ + Called after all partitions have been prepared and assembled into a + disk image. Do nothing. + """ + pass + + @classmethod + def do_configure_partition(cls, part, source_params, cr, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + native_sysroot): + """ + Called before do_prepare_partition(). Possibly prepare + configuration files of some sort. + """ + pass + + @classmethod + def do_prepare_partition(cls, part, source_params, cr, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + rootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + """ + if not bootimg_dir: + bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not bootimg_dir: + msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting\n") + + msger.debug('Bootimg dir: %s' % bootimg_dir) + + if 'file' not in source_params: + msger.error("No file specified\n") + return + + src = os.path.join(bootimg_dir, source_params['file']) + + + msger.debug('Preparing partition using image %s' % (src)) + part.prepare_rootfs_from_fs_image(cr_workdir, src, "") diff --git a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py new file mode 100644 index 0000000..ed59d85 --- /dev/null +++ b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py @@ -0,0 +1,550 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This implements the 'isoimage-isohybrid' source plugin class for 'wic' +# +# AUTHORS +# Mihaly Varga <mihaly.varga (at] ni.com> + +import os +import re +import shutil +import glob + +from wic import msger +from wic.pluginbase import SourcePlugin +from wic.utils.misc import get_custom_config +from wic.utils.oe.misc import exec_cmd, exec_native_cmd, get_bitbake_var + +class IsoImagePlugin(SourcePlugin): + """ + Create a bootable ISO image + + This plugin creates a hybrid, legacy and EFI bootable ISO image. The + generated image can be used on optical media as well as USB media. + + Legacy boot uses syslinux and EFI boot uses grub or gummiboot (not + implemented yet) as bootloader. The plugin creates the directories required + by bootloaders and populates them by creating and configuring the + bootloader files. + + Example kickstart file: + part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi, \\ + image_name= IsoImage" --ondisk cd --label LIVECD --fstype=ext2 + bootloader --timeout=10 --append=" " + + In --sourceparams "loader" specifies the bootloader used for booting in EFI + mode, while "image_name" specifies the name of the generated image. In the + example above, wic creates an ISO image named IsoImage-cd.direct (default + extension added by direct imeger plugin) and a file named IsoImage-cd.iso + """ + + name = 'isoimage-isohybrid' + + @classmethod + def do_configure_syslinux(cls, creator, cr_workdir): + """ + Create loader-specific (syslinux) config + """ + splash = os.path.join(cr_workdir, "ISO/boot/splash.jpg") + if os.path.exists(splash): + splashline = "menu background splash.jpg" + else: + splashline = "" + + bootloader = creator.ks.bootloader + + syslinux_conf = "" + syslinux_conf += "PROMPT 0\n" + syslinux_conf += "TIMEOUT %s \n" % (bootloader.timeout or 10) + syslinux_conf += "\n" + syslinux_conf += "ALLOWOPTIONS 1\n" + syslinux_conf += "SERIAL 0 115200\n" + syslinux_conf += "\n" + if splashline: + syslinux_conf += "%s\n" % splashline + syslinux_conf += "DEFAULT boot\n" + syslinux_conf += "LABEL boot\n" + + kernel = "/bzImage" + syslinux_conf += "KERNEL " + kernel + "\n" + syslinux_conf += "APPEND initrd=/initrd LABEL=boot %s\n" \ + % bootloader.append + + msger.debug("Writing syslinux config %s/ISO/isolinux/isolinux.cfg" \ + % cr_workdir) + with open("%s/ISO/isolinux/isolinux.cfg" % cr_workdir, "w") as cfg: + cfg.write(syslinux_conf) + + @classmethod + def do_configure_grubefi(cls, part, creator, cr_workdir): + """ + Create loader-specific (grub-efi) config + """ + configfile = creator.ks.bootloader.configfile + if configfile: + grubefi_conf = get_custom_config(configfile) + if grubefi_conf: + msger.debug("Using custom configuration file " + "%s for grub.cfg" % configfile) + else: + msger.error("configfile is specified but failed to " + "get it from %s." % configfile) + else: + splash = os.path.join(cr_workdir, "EFI/boot/splash.jpg") + if os.path.exists(splash): + splashline = "menu background splash.jpg" + else: + splashline = "" + + bootloader = creator.ks.bootloader + + grubefi_conf = "" + grubefi_conf += "serial --unit=0 --speed=115200 --word=8 " + grubefi_conf += "--parity=no --stop=1\n" + grubefi_conf += "default=boot\n" + grubefi_conf += "timeout=%s\n" % (bootloader.timeout or 10) + grubefi_conf += "\n" + grubefi_conf += "search --set=root --label %s " % part.label + grubefi_conf += "\n" + grubefi_conf += "menuentry 'boot'{\n" + + kernel = "/bzImage" + + grubefi_conf += "linux %s rootwait %s\n" \ + % (kernel, bootloader.append) + grubefi_conf += "initrd /initrd \n" + grubefi_conf += "}\n" + + if splashline: + grubefi_conf += "%s\n" % splashline + + msger.debug("Writing grubefi config %s/EFI/BOOT/grub.cfg" \ + % cr_workdir) + with open("%s/EFI/BOOT/grub.cfg" % cr_workdir, "w") as cfg: + cfg.write(grubefi_conf) + + @staticmethod + def _build_initramfs_path(rootfs_dir, cr_workdir): + """ + Create path for initramfs image + """ + + initrd = get_bitbake_var("INITRD") + if not initrd: + initrd_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not initrd_dir: + msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting.\n") + + image_name = get_bitbake_var("IMAGE_BASENAME") + if not image_name: + msger.error("Couldn't find IMAGE_BASENAME, exiting.\n") + + image_type = get_bitbake_var("INITRAMFS_FSTYPES") + if not image_type: + msger.error("Couldn't find INITRAMFS_FSTYPES, exiting.\n") + + machine_arch = get_bitbake_var("MACHINE_ARCH") + if not machine_arch: + msger.error("Couldn't find MACHINE_ARCH, exiting.\n") + + initrd = glob.glob('%s/%s*%s.%s' % (initrd_dir, image_name, machine_arch, image_type))[0] + + if not os.path.exists(initrd): + # Create initrd from rootfs directory + initrd = "%s/initrd.cpio.gz" % cr_workdir + initrd_dir = "%s/INITRD" % cr_workdir + shutil.copytree("%s" % rootfs_dir, \ + "%s" % initrd_dir, symlinks=True) + + if os.path.isfile("%s/init" % rootfs_dir): + shutil.copy2("%s/init" % rootfs_dir, "%s/init" % initrd_dir) + elif os.path.lexists("%s/init" % rootfs_dir): + os.symlink(os.readlink("%s/init" % rootfs_dir), \ + "%s/init" % initrd_dir) + elif os.path.isfile("%s/sbin/init" % rootfs_dir): + shutil.copy2("%s/sbin/init" % rootfs_dir, \ + "%s" % initrd_dir) + elif os.path.lexists("%s/sbin/init" % rootfs_dir): + os.symlink(os.readlink("%s/sbin/init" % rootfs_dir), \ + "%s/init" % initrd_dir) + else: + msger.error("Couldn't find or build initrd, exiting.\n") + + exec_cmd("cd %s && find . | cpio -o -H newc -R +0:+0 >./initrd.cpio " \ + % initrd_dir, as_shell=True) + exec_cmd("gzip -f -9 -c %s/initrd.cpio > %s" \ + % (cr_workdir, initrd), as_shell=True) + shutil.rmtree(initrd_dir) + + return initrd + + @classmethod + def do_stage_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + native_sysroot): + """ + Special content staging called before do_prepare_partition(). + It cheks if all necessary tools are available, if not + tries to instal them. + """ + # Make sure parted is available in native sysroot + if not os.path.isfile("%s/usr/sbin/parted" % native_sysroot): + msger.info("Building parted-native...\n") + exec_cmd("bitbake parted-native") + + # Make sure mkfs.ext2/3/4 is available in native sysroot + if not os.path.isfile("%s/sbin/mkfs.ext2" % native_sysroot): + msger.info("Building e2fsprogs-native...\n") + exec_cmd("bitbake e2fsprogs-native") + + # Make sure syslinux is available in sysroot and in native sysroot + syslinux_dir = get_bitbake_var("STAGING_DATADIR") + if not syslinux_dir: + msger.error("Couldn't find STAGING_DATADIR, exiting.\n") + if not os.path.exists("%s/syslinux" % syslinux_dir): + msger.info("Building syslinux...\n") + exec_cmd("bitbake syslinux") + if not os.path.exists("%s/syslinux" % syslinux_dir): + msger.error("Please build syslinux first\n") + + # Make sure syslinux is available in native sysroot + if not os.path.exists("%s/usr/bin/syslinux" % native_sysroot): + msger.info("Building syslinux-native...\n") + exec_cmd("bitbake syslinux-native") + + #Make sure mkisofs is available in native sysroot + if not os.path.isfile("%s/usr/bin/mkisofs" % native_sysroot): + msger.info("Building cdrtools-native...\n") + exec_cmd("bitbake cdrtools-native") + + # Make sure mkfs.vfat is available in native sysroot + if not os.path.isfile("%s/sbin/mkfs.vfat" % native_sysroot): + msger.info("Building dosfstools-native...\n") + exec_cmd("bitbake dosfstools-native") + + # Make sure mtools is available in native sysroot + if not os.path.isfile("%s/usr/bin/mcopy" % native_sysroot): + msger.info("Building mtools-native...\n") + exec_cmd("bitbake mtools-native") + + @classmethod + def do_configure_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + native_sysroot): + """ + Called before do_prepare_partition(), creates loader-specific config + """ + isodir = "%s/ISO/" % cr_workdir + + if os.path.exists(cr_workdir): + shutil.rmtree(cr_workdir) + + install_cmd = "install -d %s " % isodir + exec_cmd(install_cmd) + + # Overwrite the name of the created image + msger.debug("%s" % source_params) + if 'image_name' in source_params and \ + source_params['image_name'].strip(): + creator.name = source_params['image_name'].strip() + msger.debug("The name of the image is: %s" % creator.name) + + @classmethod + def do_prepare_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + rootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + In this case, prepare content for a bootable ISO image. + """ + + isodir = "%s/ISO" % cr_workdir + + if part.rootfs_dir is None: + if not 'ROOTFS_DIR' in rootfs_dir: + msger.error("Couldn't find --rootfs-dir, exiting.\n") + rootfs_dir = rootfs_dir['ROOTFS_DIR'] + else: + if part.rootfs_dir in rootfs_dir: + rootfs_dir = rootfs_dir[part.rootfs_dir] + elif part.rootfs_dir: + rootfs_dir = part.rootfs_dir + else: + msg = "Couldn't find --rootfs-dir=%s connection " + msg += "or it is not a valid path, exiting.\n" + msger.error(msg % part.rootfs_dir) + + if not os.path.isdir(rootfs_dir): + rootfs_dir = get_bitbake_var("IMAGE_ROOTFS") + if not os.path.isdir(rootfs_dir): + msger.error("Couldn't find IMAGE_ROOTFS, exiting.\n") + + part.rootfs_dir = rootfs_dir + + # Prepare rootfs.img + hdd_dir = get_bitbake_var("HDDDIR") + img_iso_dir = get_bitbake_var("ISODIR") + + rootfs_img = "%s/rootfs.img" % hdd_dir + if not os.path.isfile(rootfs_img): + rootfs_img = "%s/rootfs.img" % img_iso_dir + if not os.path.isfile(rootfs_img): + # check if rootfs.img is in deploydir + deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + image_name = get_bitbake_var("IMAGE_LINK_NAME") + rootfs_img = "%s/%s.%s" \ + % (deploy_dir, image_name, part.fstype) + + if not os.path.isfile(rootfs_img): + # create image file with type specified by --fstype + # which contains rootfs + du_cmd = "du -bks %s" % rootfs_dir + out = exec_cmd(du_cmd) + part.size = int(out.split()[0]) + part.extra_space = 0 + part.overhead_factor = 1.2 + part.prepare_rootfs(cr_workdir, oe_builddir, rootfs_dir, \ + native_sysroot) + rootfs_img = part.source_file + + install_cmd = "install -m 0644 %s %s/rootfs.img" \ + % (rootfs_img, isodir) + exec_cmd(install_cmd) + + # Remove the temporary file created by part.prepare_rootfs() + if os.path.isfile(part.source_file): + os.remove(part.source_file) + + # Prepare initial ramdisk + initrd = "%s/initrd" % hdd_dir + if not os.path.isfile(initrd): + initrd = "%s/initrd" % img_iso_dir + if not os.path.isfile(initrd): + initrd = cls._build_initramfs_path(rootfs_dir, cr_workdir) + + install_cmd = "install -m 0644 %s %s/initrd" \ + % (initrd, isodir) + exec_cmd(install_cmd) + + # Remove the temporary file created by _build_initramfs_path function + if os.path.isfile("%s/initrd.cpio.gz" % cr_workdir): + os.remove("%s/initrd.cpio.gz" % cr_workdir) + + # Install bzImage + install_cmd = "install -m 0644 %s/bzImage %s/bzImage" % \ + (kernel_dir, isodir) + exec_cmd(install_cmd) + + #Create bootloader for efi boot + try: + if source_params['loader'] == 'grub-efi': + # Builds grub.cfg if ISODIR didn't exist or + # didn't contains grub.cfg + bootimg_dir = img_iso_dir + if not os.path.exists("%s/EFI/BOOT" % bootimg_dir): + bootimg_dir = "%s/bootimg" % cr_workdir + if os.path.exists(bootimg_dir): + shutil.rmtree(bootimg_dir) + install_cmd = "install -d %s/EFI/BOOT" % bootimg_dir + exec_cmd(install_cmd) + + if not os.path.isfile("%s/EFI/BOOT/boot.cfg" % bootimg_dir): + cls.do_configure_grubefi(part, creator, bootimg_dir) + + # Builds bootx64.efi/bootia32.efi if ISODIR didn't exist or + # didn't contains it + target_arch = get_bitbake_var("TARGET_SYS") + if not target_arch: + msger.error("Coludn't find target architecture\n") + + if re.match("x86_64", target_arch): + grub_target = 'x86_64-efi' + grub_image = "bootx64.efi" + elif re.match('i.86', target_arch): + grub_target = 'i386-efi' + grub_image = "bootia32.efi" + else: + msger.error("grub-efi is incompatible with target %s\n" \ + % target_arch) + + if not os.path.isfile("%s/EFI/BOOT/%s" \ + % (bootimg_dir, grub_image)): + grub_path = get_bitbake_var("STAGING_LIBDIR") + if not grub_path: + msger.error("Couldn't find STAGING_LIBDIR, exiting.\n") + + grub_core = "%s/grub/%s" % (grub_path, grub_target) + if not os.path.exists(grub_core): + msger.info("Building grub-efi...\n") + exec_cmd("bitbake grub-efi") + if not os.path.exists(grub_core): + msger.error("Please build grub-efi first\n") + + grub_cmd = "grub-mkimage -p '/EFI/BOOT' " + grub_cmd += "-d %s " % grub_core + grub_cmd += "-O %s -o %s/EFI/BOOT/%s " \ + % (grub_target, bootimg_dir, grub_image) + grub_cmd += "part_gpt part_msdos ntfs ntfscomp fat ext2 " + grub_cmd += "normal chain boot configfile linux multiboot " + grub_cmd += "search efi_gop efi_uga font gfxterm gfxmenu " + grub_cmd += "terminal minicmd test iorw loadenv echo help " + grub_cmd += "reboot serial terminfo iso9660 loopback tar " + grub_cmd += "memdisk ls search_fs_uuid udf btrfs xfs lvm " + grub_cmd += "reiserfs ata " + exec_native_cmd(grub_cmd, native_sysroot) + + else: + # TODO: insert gummiboot stuff + msger.error("unrecognized bootimg-efi loader: %s" \ + % source_params['loader']) + except KeyError: + msger.error("bootimg-efi requires a loader, none specified") + + if os.path.exists("%s/EFI/BOOT" % isodir): + shutil.rmtree("%s/EFI/BOOT" % isodir) + + shutil.copytree(bootimg_dir+"/EFI/BOOT", isodir+"/EFI/BOOT") + + # If exists, remove cr_workdir/bootimg temporary folder + if os.path.exists("%s/bootimg" % cr_workdir): + shutil.rmtree("%s/bootimg" % cr_workdir) + + # Create efi.img that contains bootloader files for EFI booting + # if ISODIR didn't exist or didn't contains it + if os.path.isfile("%s/efi.img" % img_iso_dir): + install_cmd = "install -m 0644 %s/efi.img %s/efi.img" % \ + (img_iso_dir, isodir) + exec_cmd(install_cmd) + else: + du_cmd = "du -bks %s/EFI" % isodir + out = exec_cmd(du_cmd) + blocks = int(out.split()[0]) + # Add some extra space for file system overhead + blocks += 100 + msg = "Added 100 extra blocks to %s to get to %d total blocks" \ + % (part.mountpoint, blocks) + msger.debug(msg) + + # Ensure total sectors is an integral number of sectors per + # track or mcopy will complain. Sectors are 512 bytes, and we + # generate images with 32 sectors per track. This calculation is + # done in blocks, thus the mod by 16 instead of 32. + blocks += (16 - (blocks % 16)) + + # dosfs image for EFI boot + bootimg = "%s/efi.img" % isodir + + dosfs_cmd = 'mkfs.vfat -n "EFIimg" -S 512 -C %s %d' \ + % (bootimg, blocks) + exec_native_cmd(dosfs_cmd, native_sysroot) + + mmd_cmd = "mmd -i %s ::/EFI" % bootimg + exec_native_cmd(mmd_cmd, native_sysroot) + + mcopy_cmd = "mcopy -i %s -s %s/EFI/* ::/EFI/" \ + % (bootimg, isodir) + exec_native_cmd(mcopy_cmd, native_sysroot) + + chmod_cmd = "chmod 644 %s" % bootimg + exec_cmd(chmod_cmd) + + # Prepare files for legacy boot + syslinux_dir = get_bitbake_var("STAGING_DATADIR") + if not syslinux_dir: + msger.error("Couldn't find STAGING_DATADIR, exiting.\n") + + if os.path.exists("%s/isolinux" % isodir): + shutil.rmtree("%s/isolinux" % isodir) + + install_cmd = "install -d %s/isolinux" % isodir + exec_cmd(install_cmd) + + cls.do_configure_syslinux(creator, cr_workdir) + + install_cmd = "install -m 444 %s/syslinux/ldlinux.sys " % syslinux_dir + install_cmd += "%s/isolinux/ldlinux.sys" % isodir + exec_cmd(install_cmd) + + install_cmd = "install -m 444 %s/syslinux/isohdpfx.bin " % syslinux_dir + install_cmd += "%s/isolinux/isohdpfx.bin" % isodir + exec_cmd(install_cmd) + + install_cmd = "install -m 644 %s/syslinux/isolinux.bin " % syslinux_dir + install_cmd += "%s/isolinux/isolinux.bin" % isodir + exec_cmd(install_cmd) + + install_cmd = "install -m 644 %s/syslinux/ldlinux.c32 " % syslinux_dir + install_cmd += "%s/isolinux/ldlinux.c32" % isodir + exec_cmd(install_cmd) + + #create ISO image + iso_img = "%s/tempiso_img.iso" % cr_workdir + iso_bootimg = "isolinux/isolinux.bin" + iso_bootcat = "isolinux/boot.cat" + efi_img = "efi.img" + + mkisofs_cmd = "mkisofs -V %s " % part.label + mkisofs_cmd += "-o %s -U " % iso_img + mkisofs_cmd += "-J -joliet-long -r -iso-level 2 -b %s " % iso_bootimg + mkisofs_cmd += "-c %s -no-emul-boot -boot-load-size 4 " % iso_bootcat + mkisofs_cmd += "-boot-info-table -eltorito-alt-boot " + mkisofs_cmd += "-eltorito-platform 0xEF -eltorito-boot %s " % efi_img + mkisofs_cmd += "-no-emul-boot %s " % isodir + + msger.debug("running command: %s" % mkisofs_cmd) + exec_native_cmd(mkisofs_cmd, native_sysroot) + + shutil.rmtree(isodir) + + du_cmd = "du -Lbks %s" % iso_img + out = exec_cmd(du_cmd) + isoimg_size = int(out.split()[0]) + + part.size = isoimg_size + part.source_file = iso_img + + @classmethod + def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir, + bootimg_dir, kernel_dir, native_sysroot): + """ + Called after all partitions have been prepared and assembled into a + disk image. In this case, we insert/modify the MBR using isohybrid + utility for booting via BIOS from disk storage devices. + """ + + full_path = creator._full_path(workdir, disk_name, "direct") + iso_img = "%s.p1" % full_path + full_path_iso = creator._full_path(workdir, disk_name, "iso") + + isohybrid_cmd = "isohybrid -u %s" % iso_img + msger.debug("running command: %s" % \ + isohybrid_cmd) + exec_native_cmd(isohybrid_cmd, native_sysroot) + + # Replace the image created by direct plugin with the one created by + # mkisofs command. This is necessary because the iso image created by + # mkisofs has a very specific MBR is system area of the ISO image, and + # direct plugin adds and configures an another MBR. + msger.debug("Replaceing the image created by direct plugin\n") + os.remove(full_path) + shutil.copy2(iso_img, full_path_iso) + shutil.copy2(full_path_iso, full_path) + + # Remove temporary ISO file + os.remove(iso_img) diff --git a/scripts/lib/wic/plugins/source/rawcopy.py b/scripts/lib/wic/plugins/source/rawcopy.py new file mode 100644 index 0000000..7ce0cc4 --- /dev/null +++ b/scripts/lib/wic/plugins/source/rawcopy.py @@ -0,0 +1,88 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import os + +from wic import msger +from wic.pluginbase import SourcePlugin +from wic.utils.oe.misc import exec_cmd, get_bitbake_var + +class RawCopyPlugin(SourcePlugin): + """ + Populate partition content from raw image file. + """ + + name = 'rawcopy' + + @classmethod + def do_install_disk(cls, disk, disk_name, cr, workdir, oe_builddir, + bootimg_dir, kernel_dir, native_sysroot): + """ + Called after all partitions have been prepared and assembled into a + disk image. Do nothing. + """ + pass + + @classmethod + def do_configure_partition(cls, part, source_params, cr, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + native_sysroot): + """ + Called before do_prepare_partition(). Possibly prepare + configuration files of some sort. + """ + pass + + @classmethod + def do_prepare_partition(cls, part, source_params, cr, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + rootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + """ + if not bootimg_dir: + bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not bootimg_dir: + msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting\n") + + msger.debug('Bootimg dir: %s' % bootimg_dir) + + if 'file' not in source_params: + msger.error("No file specified\n") + return + + src = os.path.join(bootimg_dir, source_params['file']) + dst = os.path.join(cr_workdir, "%s.%s" % (source_params['file'], part.lineno)) + + if 'skip' in source_params: + dd_cmd = "dd if=%s of=%s ibs=%s skip=1 conv=notrunc" % \ + (src, dst, source_params['skip']) + else: + dd_cmd = "cp %s %s" % (src, dst) + exec_cmd(dd_cmd) + + # get the size in the right units for kickstart (kB) + du_cmd = "du -Lbks %s" % dst + out = exec_cmd(du_cmd) + filesize = out.split()[0] + + if int(filesize) > int(part.size): + part.size = filesize + + part.source_file = dst + diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py new file mode 100644 index 0000000..425da8b --- /dev/null +++ b/scripts/lib/wic/plugins/source/rootfs.py @@ -0,0 +1,83 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2014, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This implements the 'rootfs' source plugin class for 'wic' +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# Joao Henrique Ferreira de Freitas <joaohf (at] gmail.com> +# + +import os + +from wic import msger +from wic.pluginbase import SourcePlugin +from wic.utils.oe.misc import get_bitbake_var + +class RootfsPlugin(SourcePlugin): + """ + Populate partition content from a rootfs directory. + """ + + name = 'rootfs' + + @staticmethod + def __get_rootfs_dir(rootfs_dir): + if os.path.isdir(rootfs_dir): + return rootfs_dir + + image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir) + if not os.path.isdir(image_rootfs_dir): + msg = "No valid artifact IMAGE_ROOTFS from image named" + msg += " %s has been found at %s, exiting.\n" % \ + (rootfs_dir, image_rootfs_dir) + msger.error(msg) + + return image_rootfs_dir + + @classmethod + def do_prepare_partition(cls, part, source_params, cr, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + krootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + In this case, prepare content for legacy bios boot partition. + """ + if part.rootfs_dir is None: + if not 'ROOTFS_DIR' in krootfs_dir: + msg = "Couldn't find --rootfs-dir, exiting" + msger.error(msg) + rootfs_dir = krootfs_dir['ROOTFS_DIR'] + else: + if part.rootfs_dir in krootfs_dir: + rootfs_dir = krootfs_dir[part.rootfs_dir] + elif part.rootfs_dir: + rootfs_dir = part.rootfs_dir + else: + msg = "Couldn't find --rootfs-dir=%s connection" + msg += " or it is not a valid path, exiting" + msger.error(msg % part.rootfs_dir) + + real_rootfs_dir = cls.__get_rootfs_dir(rootfs_dir) + + part.rootfs_dir = real_rootfs_dir + part.prepare_rootfs(cr_workdir, oe_builddir, real_rootfs_dir, native_sysroot) + diff --git a/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py b/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py new file mode 100644 index 0000000..3d60e6f --- /dev/null +++ b/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py @@ -0,0 +1,177 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# This program is free software; you can distribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for mo details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# AUTHOR +# Adrian Freihofer <adrian.freihofer (at] neratec.com> +# + +import os +from wic import msger +from wic.utils import syslinux +from wic.utils import runner +from wic.utils.oe import misc +from wic.utils.errors import ImageError +from wic.pluginbase import SourcePlugin + + +# pylint: disable=no-init +class RootfsPlugin(SourcePlugin): + """ + Create root partition and install syslinux bootloader + + This plugin creates a disk image containing a bootable root partition with + syslinux installed. The filesystem is ext2/3/4, no extra boot partition is + required. + + Example kickstart file: + part / --source rootfs-pcbios-ext --ondisk sda --fstype=ext4 --label rootfs --align 1024 + bootloader --source rootfs-pcbios-ext --timeout=0 --append="rootwait rootfstype=ext4" + + The first line generates a root file system including a syslinux.cfg file + The "--source rootfs-pcbios-ext" in the second line triggers the installation + of ldlinux.sys into the image. + """ + + name = 'rootfs-pcbios-ext' + + @staticmethod + def _get_rootfs_dir(rootfs_dir): + """ + Find rootfs pseudo dir + + If rootfs_dir is a directory consider it as rootfs directory. + Otherwise ask bitbake about the IMAGE_ROOTFS directory. + """ + if os.path.isdir(rootfs_dir): + return rootfs_dir + + image_rootfs_dir = misc.get_bitbake_var("IMAGE_ROOTFS", rootfs_dir) + if not os.path.isdir(image_rootfs_dir): + msg = "No valid artifact IMAGE_ROOTFS from image named" + msg += " %s has been found at %s, exiting.\n" % \ + (rootfs_dir, image_rootfs_dir) + msger.error(msg) + + return image_rootfs_dir + + # pylint: disable=unused-argument + @classmethod + def do_configure_partition(cls, part, source_params, image_creator, + image_creator_workdir, oe_builddir, bootimg_dir, + kernel_dir, native_sysroot): + """ + Creates syslinux config in rootfs directory + + Called before do_prepare_partition() + """ + bootloader = image_creator.ks.bootloader + + syslinux_conf = "" + syslinux_conf += "PROMPT 0\n" + + syslinux_conf += "TIMEOUT " + str(bootloader.timeout) + "\n" + syslinux_conf += "ALLOWOPTIONS 1\n" + + # Derive SERIAL... line from from kernel boot parameters + syslinux_conf += syslinux.serial_console_form_kargs(options) + "\n" + + syslinux_conf += "DEFAULT linux\n" + syslinux_conf += "LABEL linux\n" + syslinux_conf += " KERNEL /boot/bzImage\n" + + syslinux_conf += " APPEND label=boot root=%s %s\n" % \ + (image_creator.rootdev, bootloader.append) + + syslinux_cfg = os.path.join(image_creator.rootfs_dir['ROOTFS_DIR'], "boot", "syslinux.cfg") + msger.debug("Writing syslinux config %s" % syslinux_cfg) + with open(syslinux_cfg, "w") as cfg: + cfg.write(syslinux_conf) + + @classmethod + def do_prepare_partition(cls, part, source_params, image_creator, + image_creator_workdir, oe_builddir, bootimg_dir, + kernel_dir, krootfs_dir, native_sysroot): + """ + Creates partition out of rootfs directory + + Prepare content for a rootfs partition i.e. create a partition + and fill it from a /rootfs dir. + Install syslinux bootloader into root partition image file + """ + def is_exe(exepath): + """Verify exepath is an executable file""" + return os.path.isfile(exepath) and os.access(exepath, os.X_OK) + + # Make sure syslinux-nomtools is available in native sysroot or fail + native_syslinux_nomtools = os.path.join(native_sysroot, "usr/bin/syslinux-nomtools") + if not is_exe(native_syslinux_nomtools): + msger.info("building syslinux-native...") + misc.exec_cmd("bitbake syslinux-native") + if not is_exe(native_syslinux_nomtools): + msger.error("Couldn't find syslinux-nomtools (%s), exiting\n" % + native_syslinux_nomtools) + + if part.rootfs is None: + if 'ROOTFS_DIR' not in krootfs_dir: + msger.error("Couldn't find --rootfs-dir, exiting") + rootfs_dir = krootfs_dir['ROOTFS_DIR'] + else: + if part.rootfs in krootfs_dir: + rootfs_dir = krootfs_dir[part.rootfs] + elif part.rootfs: + rootfs_dir = part.rootfs + else: + msg = "Couldn't find --rootfs-dir=%s connection" + msg += " or it is not a valid path, exiting" + msger.error(msg % part.rootfs) + + real_rootfs_dir = cls._get_rootfs_dir(rootfs_dir) + + part.rootfs_dir = real_rootfs_dir + part.prepare_rootfs(image_creator_workdir, oe_builddir, real_rootfs_dir, native_sysroot) + + # install syslinux into rootfs partition + syslinux_cmd = "syslinux-nomtools -d /boot -i %s" % part.source_file + misc.exec_native_cmd(syslinux_cmd, native_sysroot) + + @classmethod + def do_install_disk(cls, disk, disk_name, image_creator, workdir, oe_builddir, + bootimg_dir, kernel_dir, native_sysroot): + """ + Assemble partitions to disk image + + Called after all partitions have been prepared and assembled into a + disk image. In this case, we install the MBR. + """ + mbrfile = os.path.join(native_sysroot, "usr/share/syslinux/") + if image_creator.ptable_format == 'msdos': + mbrfile += "mbr.bin" + elif image_creator.ptable_format == 'gpt': + mbrfile += "gptmbr.bin" + else: + msger.error("Unsupported partition table: %s" % \ + image_creator.ptable_format) + + if not os.path.exists(mbrfile): + msger.error("Couldn't find %s. Has syslinux-native been baked?" % mbrfile) + + full_path = disk['disk'].device + msger.debug("Installing MBR on disk %s as %s with size %s bytes" \ + % (disk_name, full_path, disk['min_size'])) + + ret_code = runner.show(['dd', 'if=%s' % mbrfile, 'of=%s' % full_path, 'conv=notrunc']) + if ret_code != 0: + raise ImageError("Unable to set MBR to %s" % full_path) diff --git a/scripts/lib/wic/test b/scripts/lib/wic/test new file mode 100644 index 0000000..9daeafb --- /dev/null +++ b/scripts/lib/wic/test @@ -0,0 +1 @@ +test diff --git a/scripts/lib/wic/utils/__init__.py b/scripts/lib/wic/utils/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/scripts/lib/wic/utils/__init__.py diff --git a/scripts/lib/wic/utils/errors.py b/scripts/lib/wic/utils/errors.py new file mode 100644 index 0000000..d1b514d --- /dev/null +++ b/scripts/lib/wic/utils/errors.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python -tt +# +# Copyright (c) 2007 Red Hat, Inc. +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +class WicError(Exception): + pass + +class CreatorError(WicError): + pass + +class Usage(WicError): + pass + +class ImageError(WicError): + pass diff --git a/scripts/lib/wic/utils/fs_related.py b/scripts/lib/wic/utils/fs_related.py new file mode 100644 index 0000000..2e74461 --- /dev/null +++ b/scripts/lib/wic/utils/fs_related.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python -tt +# +# Copyright (c) 2007, Red Hat, Inc. +# Copyright (c) 2009, 2010, 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +from __future__ import with_statement +import os +import errno + +from wic.utils.oe.misc import exec_cmd + +def makedirs(dirname): + """A version of os.makedirs() that doesn't throw an + exception if the leaf directory already exists. + """ + try: + os.makedirs(dirname) + except OSError, err: + if err.errno != errno.EEXIST: + raise + +class Disk: + """ + Generic base object for a disk. + """ + def __init__(self, size, device=None): + self._device = device + self._size = size + + def create(self): + pass + + def cleanup(self): + pass + + def get_device(self): + return self._device + def set_device(self, path): + self._device = path + device = property(get_device, set_device) + + def get_size(self): + return self._size + size = property(get_size) + + +class DiskImage(Disk): + """ + A Disk backed by a file. + """ + def __init__(self, image_file, size): + Disk.__init__(self, size) + self.image_file = image_file + + def exists(self): + return os.path.exists(self.image_file) + + def create(self): + if self.device is not None: + return + + blocks = self.size / 1024 + if self.size - blocks * 1024: + blocks += 1 + + # create disk image + dd_cmd = "dd if=/dev/zero of=%s bs=1024 seek=%d count=1" % \ + (self.image_file, blocks) + exec_cmd(dd_cmd) + + self.device = self.image_file diff --git a/scripts/lib/wic/utils/misc.py b/scripts/lib/wic/utils/misc.py new file mode 100644 index 0000000..1415ae9 --- /dev/null +++ b/scripts/lib/wic/utils/misc.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python -tt +# +# Copyright (c) 2010, 2011 Intel Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import time +import wic.engine + +def build_name(kscfg, release=None, prefix=None, suffix=None): + """Construct and return an image name string. + + This is a utility function to help create sensible name and fslabel + strings. The name is constructed using the sans-prefix-and-extension + kickstart filename and the supplied prefix and suffix. + + kscfg -- a path to a kickstart file + release -- a replacement to suffix for image release + prefix -- a prefix to prepend to the name; defaults to None, which causes + no prefix to be used + suffix -- a suffix to append to the name; defaults to None, which causes + a YYYYMMDDHHMM suffix to be used + + Note, if maxlen is less then the len(suffix), you get to keep both pieces. + + """ + name = os.path.basename(kscfg) + idx = name.rfind('.') + if idx >= 0: + name = name[:idx] + + if release is not None: + suffix = "" + if prefix is None: + prefix = "" + if suffix is None: + suffix = time.strftime("%Y%m%d%H%M") + + if name.startswith(prefix): + name = name[len(prefix):] + + prefix = "%s-" % prefix if prefix else "" + suffix = "-%s" % suffix if suffix else "" + + ret = prefix + name + suffix + + return ret + +def find_canned(scripts_path, file_name): + """ + Find a file either by its path or by name in the canned files dir. + + Return None if not found + """ + if os.path.exists(file_name): + return file_name + + layers_canned_wks_dir = wic.engine.build_canned_image_list(scripts_path) + for canned_wks_dir in layers_canned_wks_dir: + for root, dirs, files in os.walk(canned_wks_dir): + for fname in files: + if fname == file_name: + fullpath = os.path.join(canned_wks_dir, fname) + return fullpath + +def get_custom_config(boot_file): + """ + Get the custom configuration to be used for the bootloader. + + Return None if the file can't be found. + """ + scripts_path = os.path.abspath(os.path.dirname(__file__)) + # Get the scripts path of poky + for x in range(0, 3): + scripts_path = os.path.dirname(scripts_path) + + cfg_file = find_canned(scripts_path, boot_file) + if cfg_file: + with open(cfg_file, "r") as f: + config = f.read() + return config + + return None diff --git a/scripts/lib/wic/utils/oe/__init__.py b/scripts/lib/wic/utils/oe/__init__.py new file mode 100644 index 0000000..0a81575 --- /dev/null +++ b/scripts/lib/wic/utils/oe/__init__.py @@ -0,0 +1,22 @@ +# +# OpenEmbedded wic utils library +# +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# diff --git a/scripts/lib/wic/utils/oe/misc.py b/scripts/lib/wic/utils/oe/misc.py new file mode 100644 index 0000000..81239ac --- /dev/null +++ b/scripts/lib/wic/utils/oe/misc.py @@ -0,0 +1,250 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This module provides a place to collect various wic-related utils +# for the OpenEmbedded Image Tools. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# +"""Miscellaneous functions.""" + +import os +from collections import defaultdict + +from wic import msger +from wic.utils import runner + +# executable -> recipe pairs for exec_native_cmd +NATIVE_RECIPES = {"mcopy": "mtools", + "mkdosfs": "dosfstools", + "mkfs.btrfs": "btrfs-tools", + "mkfs.ext2": "e2fsprogs", + "mkfs.ext3": "e2fsprogs", + "mkfs.ext4": "e2fsprogs", + "mkfs.vfat": "dosfstools", + "mksquashfs": "squashfs-tools", + "mkswap": "util-linux", + "parted": "parted", + "sgdisk": "gptfdisk", + "syslinux": "syslinux" + } + +def _exec_cmd(cmd_and_args, as_shell=False, catch=3): + """ + Execute command, catching stderr, stdout + + Need to execute as_shell if the command uses wildcards + """ + msger.debug("_exec_cmd: %s" % cmd_and_args) + args = cmd_and_args.split() + msger.debug(args) + + if as_shell: + ret, out = runner.runtool(cmd_and_args, catch) + else: + ret, out = runner.runtool(args, catch) + out = out.strip() + msger.debug("_exec_cmd: output for %s (rc = %d): %s" % \ + (cmd_and_args, ret, out)) + + return (ret, out) + + +def exec_cmd(cmd_and_args, as_shell=False, catch=3): + """ + Execute command, catching stderr, stdout + + Exits if rc non-zero + """ + ret, out = _exec_cmd(cmd_and_args, as_shell, catch) + + if ret != 0: + msger.error("exec_cmd: %s returned '%s' instead of 0" % \ + (cmd_and_args, ret)) + + return out + +def cmd_in_path(cmd, path): + import scriptpath + + scriptpath.add_bitbake_lib_path() + + return bb.utils.which(path, cmd) != "" or False + +def exec_native_cmd(cmd_and_args, native_sysroot, catch=3, pseudo=""): + """ + Execute native command, catching stderr, stdout + + Need to execute as_shell if the command uses wildcards + + Always need to execute native commands as_shell + """ + # The reason -1 is used is because there may be "export" commands. + args = cmd_and_args.split(';')[-1].split() + msger.debug(args) + + if pseudo: + cmd_and_args = pseudo + cmd_and_args + native_paths = \ + "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ + (native_sysroot, native_sysroot, native_sysroot) + native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ + (native_paths, cmd_and_args) + msger.debug("exec_native_cmd: %s" % cmd_and_args) + + # If the command isn't in the native sysroot say we failed. + if cmd_in_path(args[0], native_paths): + ret, out = _exec_cmd(native_cmd_and_args, True, catch) + else: + ret = 127 + + prog = args[0] + # shell command-not-found + if ret == 127 \ + or (pseudo and ret == 1 and out == "Can't find '%s' in $PATH." % prog): + msg = "A native program %s required to build the image "\ + "was not found (see details above).\n\n" % prog + recipe = NATIVE_RECIPES.get(prog) + if recipe: + msg += "Please bake it with 'bitbake %s-native' "\ + "and try again.\n" % recipe + else: + msg += "Wic failed to find a recipe to build native %s. Please "\ + "file a bug against wic.\n" % prog + msger.error(msg) + if out: + msger.debug('"%s" output: %s' % (args[0], out)) + + if ret != 0: + msger.error("exec_cmd: '%s' returned '%s' instead of 0" % \ + (cmd_and_args, ret)) + + return ret, out + +BOOTDD_EXTRA_SPACE = 16384 + +class BitbakeVars(defaultdict): + """ + Container for Bitbake variables. + """ + def __init__(self): + defaultdict.__init__(self, dict) + + # default_image and vars_dir attributes should be set from outside + self.default_image = None + self.vars_dir = None + + def _parse_line(self, line, image): + """ + Parse one line from bitbake -e output or from .env file. + Put result key-value pair into the storage. + """ + if "=" not in line: + return + try: + key, val = line.split("=") + except ValueError: + return + key = key.strip() + val = val.strip() + if key.replace('_', '').isalnum(): + self[image][key] = val.strip('"') + + def get_var(self, var, image=None): + """ + Get bitbake variable from 'bitbake -e' output or from .env file. + This is a lazy method, i.e. it runs bitbake or parses file only when + only when variable is requested. It also caches results. + """ + if not image: + image = self.default_image + + if image not in self: + if image and self.vars_dir: + fname = os.path.join(self.vars_dir, image + '.env') + if os.path.isfile(fname): + # parse .env file + with open(fname) as varsfile: + for line in varsfile: + self._parse_line(line, image) + else: + print "Couldn't get bitbake variable from %s." % fname + print "File %s doesn't exist." % fname + return + else: + # Get bitbake -e output + cmd = "bitbake -e" + if image: + cmd += " %s" % image + + log_level = msger.get_loglevel() + msger.set_loglevel('normal') + ret, lines = _exec_cmd(cmd) + msger.set_loglevel(log_level) + + if ret: + print "Couldn't get '%s' output." % cmd + print "Bitbake failed with error:\n%s\n" % lines + return + + # Parse bitbake -e output + for line in lines.split('\n'): + self._parse_line(line, image) + + # Make first image a default set of variables + images = [key for key in self if key] + if len(images) == 1: + self[None] = self[image] + + return self[image].get(var) + +# Create BB_VARS singleton +BB_VARS = BitbakeVars() + +def get_bitbake_var(var, image=None): + """ + Provide old get_bitbake_var API by wrapping + get_var method of BB_VARS singleton. + """ + return BB_VARS.get_var(var, image) + +def parse_sourceparams(sourceparams): + """ + Split sourceparams string of the form key1=val1[,key2=val2,...] + into a dict. Also accepts valueless keys i.e. without =. + + Returns dict of param key/val pairs (note that val may be None). + """ + params_dict = {} + + params = sourceparams.split(',') + if params: + for par in params: + if not par: + continue + if not '=' in par: + key = par + val = None + else: + key, val = par.split('=') + params_dict[key] = val + + return params_dict diff --git a/scripts/lib/wic/utils/partitionedfs.py b/scripts/lib/wic/utils/partitionedfs.py new file mode 100644 index 0000000..ad596d2 --- /dev/null +++ b/scripts/lib/wic/utils/partitionedfs.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python -tt +# +# Copyright (c) 2009, 2010, 2011 Intel, Inc. +# Copyright (c) 2007, 2008 Red Hat, Inc. +# Copyright (c) 2008 Daniel P. Berrange +# Copyright (c) 2008 David P. Huff +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +from wic import msger +from wic.utils.errors import ImageError +from wic.utils.oe.misc import exec_cmd, exec_native_cmd + +# Overhead of the MBR partitioning scheme (just one sector) +MBR_OVERHEAD = 1 + +# Overhead of the GPT partitioning scheme +GPT_OVERHEAD = 34 + +# Size of a sector in bytes +SECTOR_SIZE = 512 + +class Image(object): + """ + Generic base object for an image. + + An Image is a container for a set of DiskImages and associated + partitions. + """ + def __init__(self, native_sysroot=None): + self.disks = {} + self.partitions = [] + # Size of a sector used in calculations + self.sector_size = SECTOR_SIZE + self._partitions_layed_out = False + self.native_sysroot = native_sysroot + + def __add_disk(self, disk_name): + """ Add a disk 'disk_name' to the internal list of disks. Note, + 'disk_name' is the name of the disk in the target system + (e.g., sdb). """ + + if disk_name in self.disks: + # We already have this disk + return + + assert not self._partitions_layed_out + + self.disks[disk_name] = \ + {'disk': None, # Disk object + 'numpart': 0, # Number of allocate partitions + 'realpart': 0, # Number of partitions in the partition table + 'partitions': [], # Indexes to self.partitions + 'offset': 0, # Offset of next partition (in sectors) + # Minimum required disk size to fit all partitions (in bytes) + 'min_size': 0, + 'ptable_format': "msdos"} # Partition table format + + def add_disk(self, disk_name, disk_obj): + """ Add a disk object which have to be partitioned. More than one disk + can be added. In case of multiple disks, disk partitions have to be + added for each disk separately with 'add_partition()". """ + + self.__add_disk(disk_name) + self.disks[disk_name]['disk'] = disk_obj + + def __add_partition(self, part): + """ This is a helper function for 'add_partition()' which adds a + partition to the internal list of partitions. """ + + assert not self._partitions_layed_out + + self.partitions.append(part) + self.__add_disk(part['disk_name']) + + def add_partition(self, size, disk_name, mountpoint, source_file=None, fstype=None, + label=None, fsopts=None, boot=False, align=None, no_table=False, + part_type=None, uuid=None): + """ Add the next partition. Prtitions have to be added in the + first-to-last order. """ + + ks_pnum = len(self.partitions) + + # Converting kB to sectors for parted + size = size * 1024 / self.sector_size + + part = {'ks_pnum': ks_pnum, # Partition number in the KS file + 'size': size, # In sectors + 'mountpoint': mountpoint, # Mount relative to chroot + 'source_file': source_file, # partition contents + 'fstype': fstype, # Filesystem type + 'fsopts': fsopts, # Filesystem mount options + 'label': label, # Partition label + 'disk_name': disk_name, # physical disk name holding partition + 'device': None, # kpartx device node for partition + 'num': None, # Partition number + 'boot': boot, # Bootable flag + 'align': align, # Partition alignment + 'no_table' : no_table, # Partition does not appear in partition table + 'part_type' : part_type, # Partition type + 'uuid': uuid} # Partition UUID + + self.__add_partition(part) + + def layout_partitions(self, ptable_format="msdos"): + """ Layout the partitions, meaning calculate the position of every + partition on the disk. The 'ptable_format' parameter defines the + partition table format and may be "msdos". """ + + msger.debug("Assigning %s partitions to disks" % ptable_format) + + if self._partitions_layed_out: + return + + self._partitions_layed_out = True + + # Go through partitions in the order they are added in .ks file + for num in range(len(self.partitions)): + part = self.partitions[num] + + if not self.disks.has_key(part['disk_name']): + raise ImageError("No disk %s for partition %s" \ + % (part['disk_name'], part['mountpoint'])) + + if ptable_format == 'msdos' and part['part_type']: + # The --part-type can also be implemented for MBR partitions, + # in which case it would map to the 1-byte "partition type" + # filed at offset 3 of the partition entry. + raise ImageError("setting custom partition type is not " \ + "implemented for msdos partitions") + + # Get the disk where the partition is located + disk = self.disks[part['disk_name']] + disk['numpart'] += 1 + if not part['no_table']: + disk['realpart'] += 1 + disk['ptable_format'] = ptable_format + + if disk['numpart'] == 1: + if ptable_format == "msdos": + overhead = MBR_OVERHEAD + elif ptable_format == "gpt": + overhead = GPT_OVERHEAD + + # Skip one sector required for the partitioning scheme overhead + disk['offset'] += overhead + + if disk['realpart'] > 3: + # Reserve a sector for EBR for every logical partition + # before alignment is performed. + if ptable_format == "msdos": + disk['offset'] += 1 + + + if part['align']: + # If not first partition and we do have alignment set we need + # to align the partition. + # FIXME: This leaves a empty spaces to the disk. To fill the + # gaps we could enlargea the previous partition? + + # Calc how much the alignment is off. + align_sectors = disk['offset'] % (part['align'] * 1024 / self.sector_size) + + if align_sectors: + # If partition is not aligned as required, we need + # to move forward to the next alignment point + align_sectors = (part['align'] * 1024 / self.sector_size) - align_sectors + + msger.debug("Realignment for %s%s with %s sectors, original" + " offset %s, target alignment is %sK." % + (part['disk_name'], disk['numpart'], align_sectors, + disk['offset'], part['align'])) + + # increase the offset so we actually start the partition on right alignment + disk['offset'] += align_sectors + + part['start'] = disk['offset'] + disk['offset'] += part['size'] + + part['type'] = 'primary' + if not part['no_table']: + part['num'] = disk['realpart'] + else: + part['num'] = 0 + + if disk['ptable_format'] == "msdos": + if disk['realpart'] > 3: + part['type'] = 'logical' + part['num'] = disk['realpart'] + 1 + + disk['partitions'].append(num) + msger.debug("Assigned %s to %s%d, sectors range %d-%d size %d " + "sectors (%d bytes)." \ + % (part['mountpoint'], part['disk_name'], part['num'], + part['start'], part['start'] + part['size'] - 1, + part['size'], part['size'] * self.sector_size)) + + # Once all the partitions have been layed out, we can calculate the + # minumim disk sizes. + for disk in self.disks.values(): + disk['min_size'] = disk['offset'] + if disk['ptable_format'] == "gpt": + disk['min_size'] += GPT_OVERHEAD + + disk['min_size'] *= self.sector_size + + def __create_partition(self, device, parttype, fstype, start, size): + """ Create a partition on an image described by the 'device' object. """ + + # Start is included to the size so we need to substract one from the end. + end = start + size - 1 + msger.debug("Added '%s' partition, sectors %d-%d, size %d sectors" % + (parttype, start, end, size)) + + cmd = "parted -s %s unit s mkpart %s" % (device, parttype) + if fstype: + cmd += " %s" % fstype + cmd += " %d %d" % (start, end) + + return exec_native_cmd(cmd, self.native_sysroot) + + def __format_disks(self): + self.layout_partitions() + + for dev in self.disks.keys(): + disk = self.disks[dev] + msger.debug("Initializing partition table for %s" % \ + (disk['disk'].device)) + exec_native_cmd("parted -s %s mklabel %s" % \ + (disk['disk'].device, disk['ptable_format']), + self.native_sysroot) + + msger.debug("Creating partitions") + + for part in self.partitions: + if part['num'] == 0: + continue + + disk = self.disks[part['disk_name']] + if disk['ptable_format'] == "msdos" and part['num'] == 5: + # Create an extended partition (note: extended + # partition is described in MBR and contains all + # logical partitions). The logical partitions save a + # sector for an EBR just before the start of a + # partition. The extended partition must start one + # sector before the start of the first logical + # partition. This way the first EBR is inside of the + # extended partition. Since the extended partitions + # starts a sector before the first logical partition, + # add a sector at the back, so that there is enough + # room for all logical partitions. + self.__create_partition(disk['disk'].device, "extended", + None, part['start'] - 1, + disk['offset'] - part['start'] + 1) + + if part['fstype'] == "swap": + parted_fs_type = "linux-swap" + elif part['fstype'] == "vfat": + parted_fs_type = "fat32" + elif part['fstype'] == "msdos": + parted_fs_type = "fat16" + elif part['fstype'] == "ontrackdm6aux3": + parted_fs_type = "ontrackdm6aux3" + else: + # Type for ext2/ext3/ext4/btrfs + parted_fs_type = "ext2" + + # Boot ROM of OMAP boards require vfat boot partition to have an + # even number of sectors. + if part['mountpoint'] == "/boot" and part['fstype'] in ["vfat", "msdos"] \ + and part['size'] % 2: + msger.debug("Substracting one sector from '%s' partition to " \ + "get even number of sectors for the partition" % \ + part['mountpoint']) + part['size'] -= 1 + + self.__create_partition(disk['disk'].device, part['type'], + parted_fs_type, part['start'], part['size']) + + if part['part_type']: + msger.debug("partition %d: set type UID to %s" % \ + (part['num'], part['part_type'])) + exec_native_cmd("sgdisk --typecode=%d:%s %s" % \ + (part['num'], part['part_type'], + disk['disk'].device), self.native_sysroot) + + if part['uuid']: + msger.debug("partition %d: set UUID to %s" % \ + (part['num'], part['uuid'])) + exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ + (part['num'], part['uuid'], disk['disk'].device), + self.native_sysroot) + + if part['boot']: + flag_name = "legacy_boot" if disk['ptable_format'] == 'gpt' else "boot" + msger.debug("Set '%s' flag for partition '%s' on disk '%s'" % \ + (flag_name, part['num'], disk['disk'].device)) + exec_native_cmd("parted -s %s set %d %s on" % \ + (disk['disk'].device, part['num'], flag_name), + self.native_sysroot) + + # Parted defaults to enabling the lba flag for fat16 partitions, + # which causes compatibility issues with some firmware (and really + # isn't necessary). + if parted_fs_type == "fat16": + if disk['ptable_format'] == 'msdos': + msger.debug("Disable 'lba' flag for partition '%s' on disk '%s'" % \ + (part['num'], disk['disk'].device)) + exec_native_cmd("parted -s %s set %d lba off" % \ + (disk['disk'].device, part['num']), + self.native_sysroot) + + def cleanup(self): + if self.disks: + for dev in self.disks: + disk = self.disks[dev] + try: + disk['disk'].cleanup() + except: + pass + + def assemble(self, image_file): + msger.debug("Installing partitions") + + for part in self.partitions: + source = part['source_file'] + if source: + # install source_file contents into a partition + cmd = "dd if=%s of=%s bs=%d seek=%d count=%d conv=notrunc" % \ + (source, image_file, self.sector_size, + part['start'], part['size']) + exec_cmd(cmd) + + msger.debug("Installed %s in partition %d, sectors %d-%d, " + "size %d sectors" % \ + (source, part['num'], part['start'], + part['start'] + part['size'] - 1, part['size'])) + + os.rename(source, image_file + '.p%d' % part['num']) + + def create(self): + for dev in self.disks.keys(): + disk = self.disks[dev] + disk['disk'].create() + + self.__format_disks() + + return diff --git a/scripts/lib/wic/utils/runner.py b/scripts/lib/wic/utils/runner.py new file mode 100644 index 0000000..7431917 --- /dev/null +++ b/scripts/lib/wic/utils/runner.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import subprocess + +from wic import msger + +def runtool(cmdln_or_args, catch=1): + """ wrapper for most of the subprocess calls + input: + cmdln_or_args: can be both args and cmdln str (shell=True) + catch: 0, quitely run + 1, only STDOUT + 2, only STDERR + 3, both STDOUT and STDERR + return: + (rc, output) + if catch==0: the output will always None + """ + + if catch not in (0, 1, 2, 3): + # invalid catch selection, will cause exception, that's good + return None + + if isinstance(cmdln_or_args, list): + cmd = cmdln_or_args[0] + shell = False + else: + import shlex + cmd = shlex.split(cmdln_or_args)[0] + shell = True + + if catch != 3: + dev_null = os.open("/dev/null", os.O_WRONLY) + + if catch == 0: + sout = dev_null + serr = dev_null + elif catch == 1: + sout = subprocess.PIPE + serr = dev_null + elif catch == 2: + sout = dev_null + serr = subprocess.PIPE + elif catch == 3: + sout = subprocess.PIPE + serr = subprocess.STDOUT + + try: + process = subprocess.Popen(cmdln_or_args, stdout=sout, + stderr=serr, shell=shell) + (sout, serr) = process.communicate() + # combine stdout and stderr, filter None out + out = ''.join(filter(None, [sout, serr])) + except OSError, err: + if err.errno == 2: + # [Errno 2] No such file or directory + msger.error('Cannot run command: %s, lost dependency?' % cmd) + else: + raise # relay + finally: + if catch != 3: + os.close(dev_null) + + return (process.returncode, out) + +def show(cmdln_or_args): + # show all the message using msger.verbose + + rcode, out = runtool(cmdln_or_args, catch=3) + + if isinstance(cmdln_or_args, list): + cmd = ' '.join(cmdln_or_args) + else: + cmd = cmdln_or_args + + msg = 'running command: "%s"' % cmd + if out: + out = out.strip() + if out: + msg += ', with output::' + msg += '\n +----------------' + for line in out.splitlines(): + msg += '\n | %s' % line + msg += '\n +----------------' + + msger.verbose(msg) + return rcode + +def outs(cmdln_or_args, catch=1): + # get the outputs of tools + return runtool(cmdln_or_args, catch)[1].strip() + +def quiet(cmdln_or_args): + return runtool(cmdln_or_args, catch=0)[0] diff --git a/scripts/lib/wic/utils/syslinux.py b/scripts/lib/wic/utils/syslinux.py new file mode 100644 index 0000000..aace286 --- /dev/null +++ b/scripts/lib/wic/utils/syslinux.py @@ -0,0 +1,58 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# AUTHOR +# Adrian Freihofer <adrian.freihofer (at] neratec.com> + + +import re +from wic import msger + + +def serial_console_form_kargs(kernel_args): + """ + Create SERIAL... line from kernel parameters + + syslinux needs a line SERIAL port [baudrate [flowcontrol]] + in the syslinux.cfg file. The config line is generated based + on kernel boot parameters. The the parameters of the first + ttyS console are considered for syslinux config. + @param kernel_args kernel command line + @return line for syslinux config file e.g. "SERIAL 0 115200" + """ + syslinux_conf = "" + for param in kernel_args.split(): + param_match = re.match("console=ttyS([0-9]+),?([0-9]*)([noe]?)([0-9]?)(r?)", param) + if param_match: + syslinux_conf += "SERIAL " + param_match.group(1) + # baudrate + if param_match.group(2): + syslinux_conf += " " + param_match.group(2) + # parity + if param_match.group(3) and param_match.group(3) != 'n': + msger.warning("syslinux does not support parity for console. {} is ignored." + .format(param_match.group(3))) + # number of bits + if param_match.group(4) and param_match.group(4) != '8': + msger.warning("syslinux supports 8 bit console configuration only. {} is ignored." + .format(param_match.group(4))) + # flow control + if param_match.group(5) and param_match.group(5) != '': + msger.warning("syslinux console flowcontrol configuration. {} is ignored." + .format(param_match.group(5))) + break + + return syslinux_conf diff --git a/scripts/lnr b/scripts/lnr new file mode 100755 index 0000000..9dacebe --- /dev/null +++ b/scripts/lnr @@ -0,0 +1,21 @@ +#! /usr/bin/env python + +# Create a *relative* symlink, just like ln --relative does but without needing +# coreutils 8.16. + +import sys, os + +if len(sys.argv) != 3: + print "$ lnr TARGET LINK_NAME" + sys.exit(1) + +target = sys.argv[1] +linkname = sys.argv[2] + +if os.path.isabs(target): + if not os.path.isabs(linkname): + linkname = os.path.abspath(linkname) + start = os.path.dirname(linkname) + target = os.path.relpath(target, start) + +os.symlink(target, linkname) diff --git a/scripts/multilib_header_wrapper.h b/scripts/multilib_header_wrapper.h new file mode 100644 index 0000000..5a87540 --- /dev/null +++ b/scripts/multilib_header_wrapper.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2005-2011 by Wind River Systems, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include <bits/wordsize.h> + +#ifdef __WORDSIZE + +#if __WORDSIZE == 32 + +#ifdef _MIPS_SIM + +#if _MIPS_SIM == _ABIO32 +#include <ENTER_HEADER_FILENAME_HERE-32.h> +#elif _MIPS_SIM == _ABIN32 +#include <ENTER_HEADER_FILENAME_HERE-n32.h> +#else +#error "Unknown _MIPS_SIM" +#endif + +#else /* _MIPS_SIM is not defined */ +#include <ENTER_HEADER_FILENAME_HERE-32.h> +#endif + +#elif __WORDSIZE == 64 +#include <ENTER_HEADER_FILENAME_HERE-64.h> +#else +#error "Unknown __WORDSIZE detected" +#endif /* matches #if __WORDSIZE == 32 */ + +#else /* __WORDSIZE is not defined */ + +#error "__WORDSIZE is not defined" + +#endif + diff --git a/scripts/native-intercept/chown b/scripts/native-intercept/chown new file mode 100755 index 0000000..4f43271 --- /dev/null +++ b/scripts/native-intercept/chown @@ -0,0 +1,2 @@ +#! /bin/sh +echo "Intercept $0: $@ -- do nothing" diff --git a/scripts/oe-buildenv-internal b/scripts/oe-buildenv-internal new file mode 100755 index 0000000..7b04934 --- /dev/null +++ b/scripts/oe-buildenv-internal @@ -0,0 +1,121 @@ +#!/bin/sh + +# OE-Core Build Environment Setup Script +# +# Copyright (C) 2006-2011 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# It is assumed OEROOT is already defined when this is called +if [ -z "$OEROOT" ]; then + echo >&2 "Error: OEROOT is not defined!" + return 1 +fi + +if [ -z "$OE_SKIP_SDK_CHECK" ] && [ -n "$OECORE_SDK_VERSION" ]; then + echo >&2 "Error: The OE SDK/ADT was detected as already being present in this shell environment. Please use a clean shell when sourcing this environment script." + return 1 +fi + +# Make sure we're not using python v3.x. This check can't go into +# sanity.bbclass because bitbake's source code doesn't even pass +# parsing stage when used with python v3, so we catch it here so we +# can offer a meaningful error message. +py_v3_check=$((/usr/bin/env python --version 2>&1 | grep "Python 3") || true) +if [ -n "$py_v3_check" ]; then + echo >&2 "Bitbake is not compatible with python v3" + echo >&2 "Please set up python v2 as your default python interpreter" + return 1 +fi +unset py_v3_check + +# Similarly, we now have code that doesn't parse correctly with older +# versions of Python, and rather than fixing that and being eternally +# vigilant for any other new feature use, just check the version here. +py_v26_check=$(python -c 'import sys; print sys.version_info >= (2,7,3)') +if [ "$py_v26_check" != "True" ]; then + echo >&2 "BitBake requires Python 2.7.3 or later" + return 1 +fi +unset py_v26_check + +if [ -z "$BDIR" ]; then + if [ -z "$1" ]; then + BDIR="build" + else + BDIR="$1" + if [ "$BDIR" = "/" ]; then + echo >&2 "Error: / is not supported as a build directory." + return 1 + fi + + # Remove any possible trailing slashes. This is used to work around + # buggy readlink in Ubuntu 10.04 that doesn't ignore trailing slashes + # and hence "readlink -f new_dir_to_be_created/" returns empty. + BDIR=$(echo $BDIR | sed -re 's|/+$||') + + BDIR=$(readlink -f "$BDIR") + if [ -z "$BDIR" ]; then + PARENTDIR=$(dirname "$1") + echo >&2 "Error: the directory $PARENTDIR does not exist?" + return 1 + fi + fi + if [ -n "$2" ]; then + BITBAKEDIR="$2" + fi +fi +if [ "${BDIR#/}" != "$BDIR" ]; then + BUILDDIR="$BDIR" +else + BUILDDIR="$(pwd)/$BDIR" +fi +unset BDIR + +if [ -z "$BITBAKEDIR" ]; then + BITBAKEDIR="$OEROOT/bitbake$BBEXTRA" +fi + +BITBAKEDIR=$(readlink -f "$BITBAKEDIR") +BUILDDIR=$(readlink -f "$BUILDDIR") + +if [ ! -d "$BITBAKEDIR" ]; then + echo >&2 "Error: The bitbake directory ($BITBAKEDIR) does not exist! Please ensure a copy of bitbake exists at this location" + return 1 +fi + +# Make sure our paths are at the beginning of $PATH +for newpath in "$BITBAKEDIR/bin" "$OEROOT/scripts"; do + # Remove any existences of $newpath from $PATH + PATH=$(echo $PATH | sed -re "s#(^|:)$newpath(:|$)#\2#g;s#^:##") + + # Add $newpath to $PATH + PATH="$newpath:$PATH" +done +unset BITBAKEDIR newpath + +# Used by the runqemu script +export BUILDDIR +export PATH + +BB_ENV_EXTRAWHITE_OE="MACHINE DISTRO TCMODE TCLIBC HTTP_PROXY http_proxy \ +HTTPS_PROXY https_proxy FTP_PROXY ftp_proxy FTPS_PROXY ftps_proxy ALL_PROXY \ +all_proxy NO_PROXY no_proxy SSH_AGENT_PID SSH_AUTH_SOCK BB_SRCREV_POLICY \ +SDKMACHINE BB_NUMBER_THREADS BB_NO_NETWORK PARALLEL_MAKE GIT_PROXY_COMMAND \ +SOCKS5_PASSWD SOCKS5_USER SCREENDIR STAMPS_DIR" + +BB_ENV_EXTRAWHITE="$(echo $BB_ENV_EXTRAWHITE $BB_ENV_EXTRAWHITE_OE | tr ' ' '\n' | LC_ALL=C sort --unique | tr '\n' ' ')" + +export BB_ENV_EXTRAWHITE diff --git a/scripts/oe-find-native-sysroot b/scripts/oe-find-native-sysroot new file mode 100755 index 0000000..81d62b8 --- /dev/null +++ b/scripts/oe-find-native-sysroot @@ -0,0 +1,81 @@ +#!/bin/bash +# +# Find a native sysroot to use - either from an in-tree OE build or +# from a toolchain installation. It then ensures the variable +# $OECORE_NATIVE_SYSROOT is set to the sysroot's base directory, and sets +# $PSEUDO to the path of the pseudo binary. +# +# This script is intended to be run within other scripts by source'ing +# it, e.g: +# +# SYSROOT_SETUP_SCRIPT=`which oe-find-native-sysroot` +# . $SYSROOT_SETUP_SCRIPT +# +# This script will terminate execution of your calling program unless +# you set a variable $SKIP_STRICT_SYSROOT_CHECK to a non-empty string +# beforehand. +# +# Copyright (c) 2010 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +if [ "x$OECORE_NATIVE_SYSROOT" = "x" ]; then + BITBAKE=`which bitbake 2> /dev/null` + if [ "x$BITBAKE" != "x" ]; then + if [ "$UID" = "0" ]; then + # Root cannot run bitbake unless sanity checking is disabled + if [ ! -d "./conf" ]; then + echo "Error: root cannot run bitbake by default, and I cannot find a ./conf directory to be able to disable sanity checking" + exit 1 + fi + touch conf/sanity.conf + OECORE_NATIVE_SYSROOT=`bitbake -e | grep ^STAGING_DIR_NATIVE | cut -d '"' -f2` + rm -f conf/sanity.conf + else + OECORE_NATIVE_SYSROOT=`bitbake -e | grep ^STAGING_DIR_NATIVE | cut -d '"' -f2` + fi + else + echo "Error: Unable to locate bitbake command." + echo "Did you forget to source the build environment setup script?" + + if [ -z "$SKIP_STRICT_SYSROOT_CHECK" ]; then + exit 1 + fi + fi +fi + +if [ "x$OECORE_NATIVE_SYSROOT" = "x" ]; then + # This indicates that there was an error running bitbake -e that + # the user needs to be informed of + echo "There was an error running bitbake to determine STAGING_DIR_NATIVE" + echo "Here is the output from bitbake -e" + bitbake -e + exit 1 +fi + +# Set up pseudo command +if [ ! -e "$OECORE_NATIVE_SYSROOT/usr/bin/pseudo" ]; then + echo "Error: Unable to find pseudo binary in $OECORE_NATIVE_SYSROOT/usr/bin/" + + if [ "x$OECORE_DISTRO_VERSION" = "x" ]; then + echo "Have you run 'bitbake meta-ide-support'?" + else + echo "This shouldn't happen - something is wrong with your toolchain installation" + fi + + if [ -z "$SKIP_STRICT_SYSROOT_CHECK" ]; then + exit 1 + fi +fi +PSEUDO="$OECORE_NATIVE_SYSROOT/usr/bin/pseudo" diff --git a/scripts/oe-git-proxy b/scripts/oe-git-proxy new file mode 100755 index 0000000..1247902 --- /dev/null +++ b/scripts/oe-git-proxy @@ -0,0 +1,159 @@ +#!/bin/bash + +# oe-git-proxy is a simple tool to be via GIT_PROXY_COMMAND. It uses socat +# to make SOCKS5 or HTTPS proxy connections. +# It uses ALL_PROXY or all_proxy or http_proxy to determine the proxy server, +# protocol, and port. +# It uses NO_PROXY to skip using the proxy for a comma delimited list of +# hosts, host globs (*.example.com), IPs, or CIDR masks (192.168.1.0/24). It +# is known to work with both bash and dash shells. +# +# Example ALL_PROXY values: +# ALL_PROXY=socks://socks.example.com:1080 +# ALL_PROXY=https://proxy.example.com:8080 +# +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. +# +# AUTHORS +# Darren Hart <dvhart@linux.intel.com> + +# Locate the netcat binary +SOCAT=$(which socat 2>/dev/null) +if [ $? -ne 0 ]; then + echo "ERROR: socat binary not in PATH" 1>&2 + exit 1 +fi +METHOD="" + +# Test for a valid IPV4 quad with optional bitmask +valid_ipv4() { + echo $1 | egrep -q "^([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\.([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])){3}(/(3[0-2]|[1-2]?[0-9]))?$" + return $? +} + +# Convert an IPV4 address into a 32bit integer +ipv4_val() { + IP="$1" + SHIFT=24 + VAL=0 + for B in ${IP//./ }; do + VAL=$(($VAL+$(($B<<$SHIFT)))) + SHIFT=$(($SHIFT-8)) + done + echo "$VAL" +} + +# Determine if two IPs are equivalent, or if the CIDR contains the IP +match_ipv4() { + CIDR=$1 + IP=$2 + + if [ -z "${IP%%$CIDR}" ]; then + return 0 + fi + + # Determine the mask bitlength + BITS=${CIDR##*/} + [ "$BITS" != "$CIDR" ] || BITS=32 + if [ -z "$BITS" ]; then + return 1 + fi + + IPVAL=$(ipv4_val $IP) + IP2VAL=$(ipv4_val ${CIDR%%/*}) + + # OR in the unmasked bits + for i in $(seq 0 $((32-$BITS))); do + IP2VAL=$(($IP2VAL|$((1<<$i)))) + IPVAL=$(($IPVAL|$((1<<$i)))) + done + + if [ $IPVAL -eq $IP2VAL ]; then + return 0 + fi + return 1 +} + +# Test to see if GLOB matches HOST +match_host() { + HOST=$1 + GLOB=$2 + + if [ -z "${HOST%%$GLOB}" ]; then + return 0 + fi + + # Match by netmask + if valid_ipv4 $GLOB; then + HOST_IP=$(gethostip -d $HOST) + if valid_ipv4 $HOST_IP; then + match_ipv4 $GLOB $HOST_IP + if [ $? -eq 0 ]; then + return 0 + fi + fi + fi + + return 1 +} + +# If no proxy is set or needed, just connect directly +METHOD="TCP:$1:$2" + +[ -z "${ALL_PROXY}" ] && ALL_PROXY=$all_proxy +[ -z "${ALL_PROXY}" ] && ALL_PROXY=$http_proxy + +if [ -z "$ALL_PROXY" ]; then + exec $SOCAT STDIO $METHOD +fi + +# Connect directly to hosts in NO_PROXY +for H in ${NO_PROXY//,/ }; do + if match_host $1 $H; then + exec $SOCAT STDIO $METHOD + fi +done + +# Proxy is necessary, determine protocol, server, and port +# extract protocol +PROTO=${ALL_PROXY%://*} +# strip protocol:// from string +ALL_PROXY=${ALL_PROXY#*://} +# extract host & port parts: +# 1) drop username/password +PROXY=${ALL_PROXY##*@} +# 2) remove optional trailing /? +PROXY=${PROXY%%/*} +# 3) extract optional port +PORT=${PROXY##*:} +if [ "$PORT" = "$PROXY" ]; then + PORT="" +fi +# 4) remove port +PROXY=${PROXY%%:*} + +# extract username & password +PROXYAUTH="${ALL_PROXY%@*}" +[ "$PROXYAUTH" = "$ALL_PROXY" ] && PROXYAUTH= +[ -n "${PROXYAUTH}" ] && PROXYAUTH=",proxyauth=${PROXYAUTH}" + +if [ "$PROTO" = "socks" ] || [ "$PROTO" = "socks4a" ]; then + if [ -z "$PORT" ]; then + PORT="1080" + fi + METHOD="SOCKS4A:$PROXY:$1:$2,socksport=$PORT" +elif [ "$PROTO" = "socks4" ]; then + if [ -z "$PORT" ]; then + PORT="1080" + fi + METHOD="SOCKS4:$PROXY:$1:$2,socksport=$PORT" +else + # Assume PROXY (http, https, etc) + if [ -z "$PORT" ]; then + PORT="8080" + fi + METHOD="PROXY:$PROXY:$1:$2,proxyport=${PORT}${PROXYAUTH}" +fi + +exec $SOCAT STDIO "$METHOD" diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util new file mode 100755 index 0000000..a04e44d --- /dev/null +++ b/scripts/oe-pkgdata-util @@ -0,0 +1,525 @@ +#!/usr/bin/env python + +# OpenEmbedded pkgdata utility +# +# Written by: Paul Eggleton <paul.eggleton@linux.intel.com> +# +# Copyright 2012-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import sys +import os +import os.path +import fnmatch +import re +import argparse +import logging +from collections import defaultdict, OrderedDict + +scripts_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] +import scriptutils +import argparse_oe +logger = scriptutils.logger_create('pkgdatautil') + +def tinfoil_init(): + import bb.tinfoil + import logging + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(True) + + tinfoil.logger.setLevel(logging.WARNING) + return tinfoil + + +def glob(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + globs = [] + for globitem in args.glob: + globs.extend(globitem.split()) + + if not os.path.exists(args.pkglistfile): + logger.error('Unable to find package list file %s' % args.pkglistfile) + sys.exit(1) + + skipval = "-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-" + if args.exclude: + skipval += "|" + args.exclude + skipregex = re.compile(skipval) + + skippedpkgs = set() + mappedpkgs = set() + with open(args.pkglistfile, 'r') as f: + for line in f: + fields = line.rstrip().split() + if not fields: + continue + pkg = fields[0] + # We don't care about other args (used to need the package architecture but the + # new pkgdata structure avoids the need for that) + + # Skip packages for which there is no point applying globs + if skipregex.search(pkg): + logger.debug("%s -> !!" % pkg) + skippedpkgs.add(pkg) + continue + + # Skip packages that already match the globs, so if e.g. a dev package + # is already installed and thus in the list, we don't process it any further + # Most of these will be caught by skipregex already, but just in case... + already = False + for g in globs: + if fnmatch.fnmatchcase(pkg, g): + already = True + break + if already: + skippedpkgs.add(pkg) + logger.debug("%s -> !" % pkg) + continue + + # Define some functions + def revpkgdata(pkgn): + return os.path.join(args.pkgdata_dir, "runtime-reverse", pkgn) + def fwdpkgdata(pkgn): + return os.path.join(args.pkgdata_dir, "runtime", pkgn) + def readpn(pkgdata_file): + pn = "" + with open(pkgdata_file, 'r') as f: + for line in f: + if line.startswith("PN:"): + pn = line.split(': ')[1].rstrip() + return pn + def readrenamed(pkgdata_file): + renamed = "" + pn = os.path.basename(pkgdata_file) + with open(pkgdata_file, 'r') as f: + for line in f: + if line.startswith("PKG_%s:" % pn): + renamed = line.split(': ')[1].rstrip() + return renamed + + # Main processing loop + for g in globs: + mappedpkg = "" + # First just try substitution (i.e. packagename -> packagename-dev) + newpkg = g.replace("*", pkg) + revlink = revpkgdata(newpkg) + if os.path.exists(revlink): + mappedpkg = os.path.basename(os.readlink(revlink)) + fwdfile = fwdpkgdata(mappedpkg) + if os.path.exists(fwdfile): + mappedpkg = readrenamed(fwdfile) + if not os.path.exists(fwdfile + ".packaged"): + mappedpkg = "" + else: + revlink = revpkgdata(pkg) + if os.path.exists(revlink): + # Check if we can map after undoing the package renaming (by resolving the symlink) + origpkg = os.path.basename(os.readlink(revlink)) + newpkg = g.replace("*", origpkg) + fwdfile = fwdpkgdata(newpkg) + if os.path.exists(fwdfile): + mappedpkg = readrenamed(fwdfile) + else: + # That didn't work, so now get the PN, substitute that, then map in the other direction + pn = readpn(revlink) + newpkg = g.replace("*", pn) + fwdfile = fwdpkgdata(newpkg) + if os.path.exists(fwdfile): + mappedpkg = readrenamed(fwdfile) + if not os.path.exists(fwdfile + ".packaged"): + mappedpkg = "" + else: + # Package doesn't even exist... + logger.debug("%s is not a valid package!" % (pkg)) + break + + if mappedpkg: + logger.debug("%s (%s) -> %s" % (pkg, g, mappedpkg)) + mappedpkgs.add(mappedpkg) + else: + logger.debug("%s (%s) -> ?" % (pkg, g)) + + logger.debug("------") + + print("\n".join(mappedpkgs - skippedpkgs)) + +def read_value(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + packages = [] + if args.file: + with open(args.file, 'r') as f: + for line in f: + splitline = line.split() + if splitline: + packages.append(splitline[0]) + else: + for pkgitem in args.pkg: + packages.extend(pkgitem.split()) + if not packages: + logger.error("No packages specified") + sys.exit(1) + + def readvar(pkgdata_file, valuename): + val = "" + with open(pkgdata_file, 'r') as f: + for line in f: + if line.startswith(valuename + ":"): + val = line.split(': ', 1)[1].rstrip() + return val + + logger.debug("read-value('%s', '%s' '%s'" % (args.pkgdata_dir, args.valuename, packages)) + for package in packages: + pkg_split = package.split('_') + pkg_name = pkg_split[0] + logger.debug("package: '%s'" % pkg_name) + revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg_name) + logger.debug(revlink) + if os.path.exists(revlink): + mappedpkg = os.path.basename(os.readlink(revlink)) + qvar = args.valuename + if qvar == "PKGSIZE": + # append packagename + qvar = "%s_%s" % (args.valuename, mappedpkg) + # PKGSIZE is now in bytes, but we we want it in KB + pkgsize = (int(readvar(revlink, qvar)) + 1024 // 2) // 1024 + value = "%d" % pkgsize + else: + value = readvar(revlink, qvar) + if args.prefix_name: + print('%s %s' % (pkg_name, value)) + else: + print(value) + +def lookup_pkglist(pkgs, pkgdata_dir, reverse): + if reverse: + mappings = OrderedDict() + for pkg in pkgs: + revlink = os.path.join(pkgdata_dir, "runtime-reverse", pkg) + logger.debug(revlink) + if os.path.exists(revlink): + mappings[pkg] = os.path.basename(os.readlink(revlink)) + else: + mappings = defaultdict(list) + for pkg in pkgs: + pkgfile = os.path.join(pkgdata_dir, 'runtime', pkg) + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0] == 'PKG_%s' % pkg: + mappings[pkg].append(fields[1]) + break + return mappings + +def lookup_pkg(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + pkgs = [] + for pkgitem in args.pkg: + pkgs.extend(pkgitem.split()) + + mappings = lookup_pkglist(pkgs, args.pkgdata_dir, args.reverse) + + if len(mappings) < len(pkgs): + missing = list(set(pkgs) - set(mappings.keys())) + logger.error("The following packages could not be found: %s" % ', '.join(missing)) + sys.exit(1) + + if args.reverse: + items = mappings.values() + else: + items = [] + for pkg in pkgs: + items.extend(mappings.get(pkg, [])) + + print('\n'.join(items)) + +def lookup_recipe(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + pkgs = [] + for pkgitem in args.pkg: + pkgs.extend(pkgitem.split()) + + mappings = defaultdict(list) + for pkg in pkgs: + pkgfile = os.path.join(args.pkgdata_dir, 'runtime-reverse', pkg) + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0] == 'PN': + mappings[pkg].append(fields[1]) + break + if len(mappings) < len(pkgs): + missing = list(set(pkgs) - set(mappings.keys())) + logger.error("The following packages could not be found: %s" % ', '.join(missing)) + sys.exit(1) + + items = [] + for pkg in pkgs: + items.extend(mappings.get(pkg, [])) + print('\n'.join(items)) + +def get_recipe_pkgs(pkgdata_dir, recipe, unpackaged): + recipedatafile = os.path.join(pkgdata_dir, recipe) + if not os.path.exists(recipedatafile): + logger.error("Unable to find packaged recipe with name %s" % recipe) + sys.exit(1) + packages = [] + with open(recipedatafile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0] == 'PACKAGES': + packages = fields[1].split() + break + + if not unpackaged: + pkglist = [] + for pkg in packages: + if os.path.exists(os.path.join(pkgdata_dir, 'runtime', '%s.packaged' % pkg)): + pkglist.append(pkg) + return pkglist + else: + return packages + +def list_pkgs(args): + found = False + + def matchpkg(pkg): + if args.pkgspec: + matched = False + for pkgspec in args.pkgspec: + if fnmatch.fnmatchcase(pkg, pkgspec): + matched = True + break + if not matched: + return False + if not args.unpackaged: + if args.runtime: + revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg) + if os.path.exists(revlink): + # We're unlikely to get here if the package was not packaged, but just in case + # we add the symlinks for unpackaged files in the future + mappedpkg = os.path.basename(os.readlink(revlink)) + if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % mappedpkg)): + return False + else: + return False + else: + if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % pkg)): + return False + return True + + if args.recipe: + packages = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged) + + if args.runtime: + pkglist = [] + runtime_pkgs = lookup_pkglist(packages, args.pkgdata_dir, False) + for rtpkgs in runtime_pkgs.values(): + pkglist.extend(rtpkgs) + else: + pkglist = packages + + for pkg in pkglist: + if matchpkg(pkg): + found = True + print("%s" % pkg) + else: + if args.runtime: + searchdir = 'runtime-reverse' + else: + searchdir = 'runtime' + + for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, searchdir)): + for fn in files: + if fn.endswith('.packaged'): + continue + if matchpkg(fn): + found = True + print("%s" % fn) + if not found: + if args.pkgspec: + logger.error("Unable to find any package matching %s" % args.pkgspec) + else: + logger.error("No packages found") + sys.exit(1) + +def list_pkg_files(args): + import json + + if args.recipe: + if args.pkg: + logger.error("list-pkg-files: If -p/--recipe is specified then a package name cannot be specified") + sys.exit(1) + recipepkglist = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged) + if args.runtime: + pkglist = [] + runtime_pkgs = lookup_pkglist(recipepkglist, args.pkgdata_dir, False) + for rtpkgs in runtime_pkgs.values(): + pkglist.extend(rtpkgs) + else: + pkglist = recipepkglist + else: + if not args.pkg: + logger.error("list-pkg-files: If -p/--recipe is not specified then at least one package name must be specified") + sys.exit(1) + pkglist = args.pkg + + for pkg in sorted(pkglist): + print("%s:" % pkg) + if args.runtime: + pkgdatafile = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg) + if not os.path.exists(pkgdatafile): + if args.recipe: + # This package was empty and thus never packaged, ignore + continue + logger.error("Unable to find any built runtime package named %s" % pkg) + sys.exit(1) + else: + pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", pkg) + if not os.path.exists(pkgdatafile): + logger.error("Unable to find any built recipe-space package named %s" % pkg) + sys.exit(1) + + with open(pkgdatafile, 'r') as f: + found = False + for line in f: + if line.startswith('FILES_INFO:'): + found = True + val = line.split(':', 1)[1].strip() + dictval = json.loads(val) + for fullpth in sorted(dictval): + print("\t%s" % fullpth) + break + if not found: + logger.error("Unable to find FILES_INFO entry in %s" % pkgdatafile) + sys.exit(1) + +def find_path(args): + import json + + found = False + for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, 'runtime')): + for fn in files: + with open(os.path.join(root,fn)) as f: + for line in f: + if line.startswith('FILES_INFO:'): + val = line.split(':', 1)[1].strip() + dictval = json.loads(val) + for fullpth in dictval.keys(): + if fnmatch.fnmatchcase(fullpth, args.targetpath): + found = True + print("%s: %s" % (fn, fullpth)) + break + if not found: + logger.error("Unable to find any package producing path %s" % args.targetpath) + sys.exit(1) + + +def main(): + parser = argparse_oe.ArgumentParser(description="OpenEmbedded pkgdata tool - queries the pkgdata files written out during do_package", + epilog="Use %(prog)s <subcommand> --help to get help on a specific command") + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-p', '--pkgdata-dir', help='Path to pkgdata directory (determined automatically if not specified)') + subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') + + parser_lookup_pkg = subparsers.add_parser('lookup-pkg', + help='Translate between recipe-space package names and runtime package names', + description='Looks up the specified recipe-space package name(s) to see what the final runtime package name is (e.g. glibc becomes libc6), or with -r/--reverse looks up the other way.') + parser_lookup_pkg.add_argument('pkg', nargs='+', help='Package name to look up') + parser_lookup_pkg.add_argument('-r', '--reverse', help='Switch to looking up recipe-space package names from runtime package names', action='store_true') + parser_lookup_pkg.set_defaults(func=lookup_pkg) + + parser_list_pkgs = subparsers.add_parser('list-pkgs', + help='List packages', + description='Lists packages that have been built') + parser_list_pkgs.add_argument('pkgspec', nargs='*', help='Package name to search for (wildcards * ? allowed, use quotes to avoid shell expansion)') + parser_list_pkgs.add_argument('-r', '--runtime', help='Show runtime package names instead of recipe-space package names', action='store_true') + parser_list_pkgs.add_argument('-p', '--recipe', help='Limit to packages produced by the specified recipe') + parser_list_pkgs.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages', action='store_true') + parser_list_pkgs.set_defaults(func=list_pkgs) + + parser_list_pkg_files = subparsers.add_parser('list-pkg-files', + help='List files within a package', + description='Lists files included in one or more packages') + parser_list_pkg_files.add_argument('pkg', nargs='*', help='Package name to report on (if -p/--recipe is not specified)') + parser_list_pkg_files.add_argument('-r', '--runtime', help='Specified package(s) are runtime package names instead of recipe-space package names', action='store_true') + parser_list_pkg_files.add_argument('-p', '--recipe', help='Report on all packages produced by the specified recipe') + parser_list_pkg_files.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages (only useful with -p/--recipe)', action='store_true') + parser_list_pkg_files.set_defaults(func=list_pkg_files) + + parser_lookup_recipe = subparsers.add_parser('lookup-recipe', + help='Find recipe producing one or more packages', + description='Looks up the specified runtime package(s) to see which recipe they were produced by') + parser_lookup_recipe.add_argument('pkg', nargs='+', help='Runtime package name to look up') + parser_lookup_recipe.set_defaults(func=lookup_recipe) + + parser_find_path = subparsers.add_parser('find-path', + help='Find package providing a target path', + description='Finds the recipe-space package providing the specified target path') + parser_find_path.add_argument('targetpath', help='Path to find (wildcards * ? allowed, use quotes to avoid shell expansion)') + parser_find_path.set_defaults(func=find_path) + + parser_read_value = subparsers.add_parser('read-value', + help='Read any pkgdata value for one or more packages', + description='Reads the named value from the pkgdata files for the specified packages') + parser_read_value.add_argument('valuename', help='Name of the value to look up') + parser_read_value.add_argument('pkg', nargs='*', help='Runtime package name to look up') + parser_read_value.add_argument('-f', '--file', help='Read package names from the specified file (one per line, first field only)') + parser_read_value.add_argument('-n', '--prefix-name', help='Prefix output with package name', action='store_true') + parser_read_value.set_defaults(func=read_value) + + parser_glob = subparsers.add_parser('glob', + help='Expand package name glob expression', + description='Expands one or more glob expressions over the packages listed in pkglistfile') + parser_glob.add_argument('pkglistfile', help='File listing packages (one package name per line)') + parser_glob.add_argument('glob', nargs="+", help='Glob expression for package names, e.g. *-dev') + parser_glob.add_argument('-x', '--exclude', help='Exclude packages matching specified regex from the glob operation') + parser_glob.set_defaults(func=glob) + + + args = parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + + if not args.pkgdata_dir: + import scriptpath + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + sys.exit(1) + logger.debug('Found bitbake path: %s' % bitbakepath) + tinfoil = tinfoil_init() + args.pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) + logger.debug('Value of PKGDATA_DIR is "%s"' % args.pkgdata_dir) + if not args.pkgdata_dir: + logger.error('Unable to determine pkgdata directory from PKGDATA_DIR') + sys.exit(1) + + if not os.path.exists(args.pkgdata_dir): + logger.error('Unable to find pkgdata directory %s' % args.pkgdata_dir) + sys.exit(1) + + ret = args.func(args) + + return ret + + +if __name__ == "__main__": + main() diff --git a/scripts/oe-publish-sdk b/scripts/oe-publish-sdk new file mode 100755 index 0000000..55872f2 --- /dev/null +++ b/scripts/oe-publish-sdk @@ -0,0 +1,155 @@ +#!/usr/bin/env python + +# OpenEmbedded SDK publishing tool + +# Copyright (C) 2015-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import os +import argparse +import glob +import re +import subprocess +import logging +import shutil +import errno + +scripts_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] +import scriptutils +import argparse_oe +logger = scriptutils.logger_create('sdktool') + +def mkdir(d): + try: + os.makedirs(d) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + +def publish(args): + logger.debug("In publish function") + target_sdk = args.sdk + destination = args.dest + logger.debug("target_sdk = %s, update_server = %s" % (target_sdk, destination)) + sdk_basename = os.path.basename(target_sdk) + + # Ensure the SDK exists + if not os.path.exists(target_sdk): + logger.error("Specified SDK %s doesn't exist" % target_sdk) + return -1 + if os.path.isdir(target_sdk): + logger.error("%s is a directory - expected path to SDK installer file" % target_sdk) + return -1 + + if ':' in destination: + is_remote = True + host, destdir = destination.split(':') + dest_sdk = os.path.join(destdir, sdk_basename) + else: + is_remote = False + dest_sdk = os.path.join(destination, sdk_basename) + destdir = destination + + # Making sure the directory exists + logger.debug("Making sure the destination directory exists") + if not is_remote: + mkdir(destination) + else: + cmd = "ssh %s 'mkdir -p %s'" % (host, destdir) + ret = subprocess.call(cmd, shell=True) + if ret != 0: + logger.error("Making directory %s on %s failed" % (destdir, host)) + return ret + + # Copying the SDK to the destination + logger.info("Copying the SDK to destination") + if not is_remote: + if os.path.exists(dest_sdk): + os.remove(dest_sdk) + if (os.stat(target_sdk).st_dev == os.stat(destination).st_dev): + os.link(target_sdk, dest_sdk) + else: + shutil.copy(target_sdk, dest_sdk) + else: + cmd = "scp %s %s" % (target_sdk, destination) + ret = subprocess.call(cmd, shell=True) + if ret != 0: + logger.error("scp %s %s failed" % (target_sdk, destination)) + return ret + + # Unpack the SDK + logger.info("Unpacking SDK") + if not is_remote: + cmd = "sh %s -p -y -d %s" % (dest_sdk, destination) + ret = subprocess.call(cmd, shell=True) + if ret == 0: + logger.info('Successfully unpacked %s to %s' % (dest_sdk, destination)) + os.remove(dest_sdk) + else: + logger.error('Failed to unpack %s to %s' % (dest_sdk, destination)) + return ret + else: + cmd = "ssh %s 'sh %s -p -y -d %s && rm -f %s'" % (host, dest_sdk, destdir, dest_sdk) + ret = subprocess.call(cmd, shell=True) + if ret == 0: + logger.info('Successfully unpacked %s to %s' % (dest_sdk, destdir)) + else: + logger.error('Failed to unpack %s to %s' % (dest_sdk, destdir)) + return ret + + # Setting up the git repo + if not is_remote: + cmd = 'set -e; mkdir -p %s/layers; cd %s/layers; if [ ! -e .git ]; then git init .; mv .git/hooks/post-update.sample .git/hooks/post-update; echo "*.pyc\n*.pyo" > .gitignore; fi; git add -A .; git config user.email "oe@oe.oe" && git config user.name "OE" && git commit -q -m "init repo" || true; git update-server-info' % (destination, destination) + else: + cmd = "ssh %s 'set -e; mkdir -p %s/layers; cd %s/layers; if [ ! -e .git ]; then git init .; mv .git/hooks/post-update.sample .git/hooks/post-update; echo '*.pyc\n*.pyo' > .gitignore; fi; git add -A .; git config user.email 'oe@oe.oe' && git config user.name 'OE' && git commit -q -m \"init repo\" || true; git update-server-info'" % (host, destdir, destdir) + ret = subprocess.call(cmd, shell=True) + if ret == 0: + logger.info('SDK published successfully') + else: + logger.error('Failed to set up layer git repo') + return ret + + +def main(): + parser = argparse_oe.ArgumentParser(description="OpenEmbedded extensible SDK publishing tool - writes server-side data to support the extensible SDK update process to a specified location") + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') + + parser.add_argument('sdk', help='Extensible SDK to publish (path to .sh installer file)') + parser.add_argument('dest', help='Destination to publish SDK to; can be local path or remote in the form of user@host:/path (in the latter case ssh/scp will be used).') + + parser.set_defaults(func=publish) + + args = parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + elif args.quiet: + logger.setLevel(logging.ERROR) + + ret = args.func(args) + return ret + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/scripts/oe-selftest b/scripts/oe-selftest new file mode 100755 index 0000000..5e23ef0 --- /dev/null +++ b/scripts/oe-selftest @@ -0,0 +1,636 @@ +#!/usr/bin/env python + +# Copyright (c) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# DESCRIPTION +# This script runs tests defined in meta/lib/oeqa/selftest/ +# It's purpose is to automate the testing of different bitbake tools. +# To use it you just need to source your build environment setup script and +# add the meta-selftest layer to your BBLAYERS. +# Call the script as: "oe-selftest -a" to run all the tests in meta/lib/oeqa/selftest/ +# Call the script as: "oe-selftest -r <module>.<Class>.<method>" to run just a single test +# E.g: "oe-selftest -r bblayers.BitbakeLayers" will run just the BitbakeLayers class from meta/lib/oeqa/selftest/bblayers.py + + +import os +import sys +import unittest +import logging +import argparse +import subprocess +import time as t +import re +import fnmatch + +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib') +import scriptpath +scriptpath.add_bitbake_lib_path() +scriptpath.add_oe_lib_path() +import argparse_oe + +import oeqa.selftest +import oeqa.utils.ftools as ftools +from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer +from oeqa.selftest.base import oeSelfTest, get_available_machines + +def logger_create(): + log_file = "oe-selftest-" + t.strftime("%Y-%m-%d_%H:%M:%S") + ".log" + if os.path.exists("oe-selftest.log"): os.remove("oe-selftest.log") + os.symlink(log_file, "oe-selftest.log") + + log = logging.getLogger("selftest") + log.setLevel(logging.DEBUG) + + fh = logging.FileHandler(filename=log_file, mode='w') + fh.setLevel(logging.DEBUG) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + ch.setFormatter(formatter) + + log.addHandler(fh) + log.addHandler(ch) + + return log + +log = logger_create() + +def get_args_parser(): + description = "Script that runs unit tests agains bitbake and other Yocto related tools. The goal is to validate tools functionality and metadata integrity. Refer to https://wiki.yoctoproject.org/wiki/Oe-selftest for more information." + parser = argparse_oe.ArgumentParser(description=description) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-r', '--run-tests', required=False, action='store', nargs='*', dest="run_tests", default=None, help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>') + group.add_argument('-a', '--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False, help='Run all (unhidden) tests') + group.add_argument('-m', '--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.') + group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.') + parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing") + parser.add_argument('--coverage-source', dest="coverage_source", nargs="+", help="Specifiy the directories to take coverage from") + parser.add_argument('--coverage-include', dest="coverage_include", nargs="+", help="Specify extra patterns to include into the coverage measurement") + parser.add_argument('--coverage-omit', dest="coverage_omit", nargs="+", help="Specify with extra patterns to exclude from the coverage measurement") + group.add_argument('--run-tests-by', required=False, dest='run_tests_by', default=False, nargs='*', + help='run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>') + group.add_argument('--list-tests-by', required=False, dest='list_tests_by', default=False, nargs='*', + help='list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>') + group.add_argument('-l', '--list-tests', required=False, action="store_true", dest="list_tests", default=False, + help='List all available tests.') + group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true", + help='List all tags that have been set to test cases.') + parser.add_argument('--machine', required=False, dest='machine', choices=['random', 'all'], default=None, + help='Run tests on different machines (random/all).') + return parser + + +def preflight_check(): + + log.info("Checking that everything is in order before running the tests") + + if not os.environ.get("BUILDDIR"): + log.error("BUILDDIR isn't set. Did you forget to source your build environment setup script?") + return False + + builddir = os.environ.get("BUILDDIR") + if os.getcwd() != builddir: + log.info("Changing cwd to %s" % builddir) + os.chdir(builddir) + + if not "meta-selftest" in get_bb_var("BBLAYERS"): + log.error("You don't seem to have the meta-selftest layer in BBLAYERS") + return False + + log.info("Running bitbake -p") + runCmd("bitbake -p") + + return True + +def add_include(): + builddir = os.environ.get("BUILDDIR") + if "#include added by oe-selftest.py" \ + not in ftools.read_file(os.path.join(builddir, "conf/local.conf")): + log.info("Adding: \"include selftest.inc\" in local.conf") + ftools.append_file(os.path.join(builddir, "conf/local.conf"), \ + "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc") + + if "#include added by oe-selftest.py" \ + not in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")): + log.info("Adding: \"include bblayers.inc\" in bblayers.conf") + ftools.append_file(os.path.join(builddir, "conf/bblayers.conf"), \ + "\n#include added by oe-selftest.py\ninclude bblayers.inc") + +def remove_include(): + builddir = os.environ.get("BUILDDIR") + if builddir is None: + return + if "#include added by oe-selftest.py" \ + in ftools.read_file(os.path.join(builddir, "conf/local.conf")): + log.info("Removing the include from local.conf") + ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \ + "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc") + + if "#include added by oe-selftest.py" \ + in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")): + log.info("Removing the include from bblayers.conf") + ftools.remove_from_file(os.path.join(builddir, "conf/bblayers.conf"), \ + "\n#include added by oe-selftest.py\ninclude bblayers.inc") + +def remove_inc_files(): + try: + os.remove(os.path.join(os.environ.get("BUILDDIR"), "conf/selftest.inc")) + for root, _, files in os.walk(get_test_layer()): + for f in files: + if f == 'test_recipe.inc': + os.remove(os.path.join(root, f)) + except (AttributeError, OSError,) as e: # AttributeError may happen if BUILDDIR is not set + pass + + for incl_file in ['conf/bblayers.inc', 'conf/machine.inc']: + try: + os.remove(os.path.join(os.environ.get("BUILDDIR"), incl_file)) + except: + pass + + +def get_tests_modules(include_hidden=False): + modules_list = list() + for modules_path in oeqa.selftest.__path__: + for (p, d, f) in os.walk(modules_path): + files = sorted([f for f in os.listdir(p) if f.endswith('.py') and not (f.startswith('_') and not include_hidden) and not f.startswith('__') and f != 'base.py']) + for f in files: + submodules = p.split("selftest")[-1] + module = "" + if submodules: + module = 'oeqa.selftest' + submodules.replace("/",".") + "." + f.split('.py')[0] + else: + module = 'oeqa.selftest.' + f.split('.py')[0] + if module not in modules_list: + modules_list.append(module) + return modules_list + + +def get_tests(exclusive_modules=[], include_hidden=False): + test_modules = list() + for x in exclusive_modules: + test_modules.append('oeqa.selftest.' + x) + if not test_modules: + inc_hidden = include_hidden + test_modules = get_tests_modules(inc_hidden) + + return test_modules + + +class Tc: + def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None): + self.tcname = tcname + self.tcclass = tcclass + self.tcmodule = tcmodule + self.tcid = tcid + # A test case can have multiple tags (as tuples) otherwise str will suffice + self.tctag = tctag + self.fullpath = '.'.join(['oeqa', 'selftest', tcmodule, tcclass, tcname]) + + +def get_tests_from_module(tmod): + tlist = [] + prefix = 'oeqa.selftest.' + + try: + import importlib + modlib = importlib.import_module(tmod) + for mod in vars(modlib).values(): + if isinstance(mod, type(oeSelfTest)) and issubclass(mod, oeSelfTest) and mod is not oeSelfTest: + for test in dir(mod): + if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'): + # Get test case id and feature tag + # NOTE: if testcase decorator or feature tag not set will throw error + try: + tid = vars(mod)[test].test_case + except: + print 'DEBUG: tc id missing for ' + str(test) + tid = None + try: + ttag = vars(mod)[test].tag__feature + except: + # print 'DEBUG: feature tag missing for ' + str(test) + ttag = None + + # NOTE: for some reason lstrip() doesn't work for mod.__module__ + tlist.append(Tc(test, mod.__name__, mod.__module__.replace(prefix, ''), tid, ttag)) + except: + pass + + return tlist + + +def get_all_tests(): + # Get all the test modules (except the hidden ones) + testlist = [] + tests_modules = get_tests_modules() + # Get all the tests from modules + for tmod in sorted(tests_modules): + testlist += get_tests_from_module(tmod) + return testlist + + +def get_testsuite_by(criteria, keyword): + # Get a testsuite based on 'keyword' + # criteria: name, class, module, id, tag + # keyword: a list of tests, classes, modules, ids, tags + + ts = [] + all_tests = get_all_tests() + + def get_matches(values): + # Get an item and return the ones that match with keyword(s) + # values: the list of items (names, modules, classes...) + result = [] + remaining = values[:] + for key in keyword: + if key in remaining: + # Regular matching of exact item + result.append(key) + remaining.remove(key) + else: + # Wildcard matching + pattern = re.compile(fnmatch.translate(r"%s" % key)) + added = [x for x in remaining if pattern.match(x)] + result.extend(added) + remaining = [x for x in remaining if x not in added] + + return result + + if criteria == 'name': + names = get_matches([ tc.tcname for tc in all_tests ]) + ts = [ tc for tc in all_tests if tc.tcname in names ] + + elif criteria == 'class': + classes = get_matches([ tc.tcclass for tc in all_tests ]) + ts = [ tc for tc in all_tests if tc.tcclass in classes ] + + elif criteria == 'module': + modules = get_matches([ tc.tcmodule for tc in all_tests ]) + ts = [ tc for tc in all_tests if tc.tcmodule in modules ] + + elif criteria == 'id': + ids = get_matches([ str(tc.tcid) for tc in all_tests ]) + ts = [ tc for tc in all_tests if str(tc.tcid) in ids ] + + elif criteria == 'tag': + values = set() + for tc in all_tests: + # tc can have multiple tags (as tuple) otherwise str will suffice + if isinstance(tc.tctag, tuple): + values |= { str(tag) for tag in tc.tctag } + else: + values.add(str(tc.tctag)) + + tags = get_matches(list(values)) + + for tc in all_tests: + for tag in tags: + if isinstance(tc.tctag, tuple) and tag in tc.tctag: + ts.append(tc) + elif tag == tc.tctag: + ts.append(tc) + + # Remove duplicates from the list + ts = list(set(ts)) + + return ts + + +def list_testsuite_by(criteria, keyword): + # Get a testsuite based on 'keyword' + # criteria: name, class, module, id, tag + # keyword: a list of tests, classes, modules, ids, tags + + ts = sorted([ (tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule) for tc in get_testsuite_by(criteria, keyword) ]) + + print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module') + print '_' * 150 + for t in ts: + if isinstance(t[1], (tuple, list)): + print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4]) + else: + print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t + print '_' * 150 + print 'Filtering by:\t %s' % criteria + print 'Looking for:\t %s' % ', '.join(str(x) for x in keyword) + print 'Total found:\t %s' % len(ts) + + +def list_tests(): + # List all available oe-selftest tests + + ts = get_all_tests() + + print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module') + print '_' * 150 + for t in ts: + if isinstance(t.tctag, (tuple, list)): + print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t.tcid, ', '.join(t.tctag), t.tcname, t.tcclass, t.tcmodule) + else: + print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t.tcid, t.tctag, t.tcname, t.tcclass, t.tcmodule) + print '_' * 150 + print 'Total found:\t %s' % len(ts) + + +def list_tags(): + # Get all tags set to test cases + # This is useful when setting tags to test cases + # The list of tags should be kept as minimal as possible + tags = set() + all_tests = get_all_tests() + + for tc in all_tests: + if isinstance(tc.tctag, (tuple, list)): + tags.update(set(tc.tctag)) + else: + tags.add(tc.tctag) + + print 'Tags:\t%s' % ', '.join(str(x) for x in tags) + +def coverage_setup(coverage_source, coverage_include, coverage_omit): + """ Set up the coverage measurement for the testcases to be run """ + import datetime + import subprocess + builddir = os.environ.get("BUILDDIR") + pokydir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + curcommit= subprocess.check_output(["git", "--git-dir", os.path.join(pokydir, ".git"), "rev-parse", "HEAD"]) + coveragerc = "%s/.coveragerc" % builddir + data_file = "%s/.coverage." % builddir + data_file += datetime.datetime.now().strftime('%Y%m%dT%H%M%S') + if os.path.isfile(data_file): + os.remove(data_file) + with open(coveragerc, 'w') as cps: + cps.write("# Generated with command '%s'\n" % " ".join(sys.argv)) + cps.write("# HEAD commit %s\n" % curcommit.strip()) + cps.write("[run]\n") + cps.write("data_file = %s\n" % data_file) + cps.write("branch = True\n") + # Measure just BBLAYERS, scripts and bitbake folders + cps.write("source = \n") + if coverage_source: + for directory in coverage_source: + if not os.path.isdir(directory): + log.warn("Directory %s is not valid.", directory) + cps.write(" %s\n" % directory) + else: + for layer in get_bb_var('BBLAYERS').split(): + cps.write(" %s\n" % layer) + cps.write(" %s\n" % os.path.dirname(os.path.realpath(__file__))) + cps.write(" %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake')) + + if coverage_include: + cps.write("include = \n") + for pattern in coverage_include: + cps.write(" %s\n" % pattern) + if coverage_omit: + cps.write("omit = \n") + for pattern in coverage_omit: + cps.write(" %s\n" % pattern) + + return coveragerc + +def coverage_report(): + """ Loads the coverage data gathered and reports it back """ + try: + # Coverage4 uses coverage.Coverage + from coverage import Coverage + except: + # Coverage under version 4 uses coverage.coverage + from coverage import coverage as Coverage + + import cStringIO as StringIO + from coverage.misc import CoverageException + + cov_output = StringIO.StringIO() + # Creating the coverage data with the setting from the configuration file + cov = Coverage(config_file = os.environ.get('COVERAGE_PROCESS_START')) + try: + # Load data from the data file specified in the configuration + cov.load() + # Store report data in a StringIO variable + cov.report(file = cov_output, show_missing=False) + log.info("\n%s" % cov_output.getvalue()) + except CoverageException as e: + # Show problems with the reporting. Since Coverage4 not finding any data to report raises an exception + log.warn("%s" % str(e)) + finally: + cov_output.close() + + +def main(): + parser = get_args_parser() + args = parser.parse_args() + + # Add <layer>/lib to sys.path, so layers can add selftests + log.info("Running bitbake -e to get BBPATH") + bbpath = get_bb_var('BBPATH').split(':') + layer_libdirs = [p for p in (os.path.join(l, 'lib') for l in bbpath) if os.path.exists(p)] + sys.path.extend(layer_libdirs) + reload(oeqa.selftest) + + if args.run_tests_by and len(args.run_tests_by) >= 2: + valid_options = ['name', 'class', 'module', 'id', 'tag'] + if args.run_tests_by[0] not in valid_options: + print '--run-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.run_tests_by[0] + return 1 + else: + criteria = args.run_tests_by[0] + keyword = args.run_tests_by[1:] + ts = sorted([ tc.fullpath for tc in get_testsuite_by(criteria, keyword) ]) + + if args.list_tests_by and len(args.list_tests_by) >= 2: + valid_options = ['name', 'class', 'module', 'id', 'tag'] + if args.list_tests_by[0] not in valid_options: + print '--list-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.list_tests_by[0] + return 1 + else: + criteria = args.list_tests_by[0] + keyword = args.list_tests_by[1:] + list_testsuite_by(criteria, keyword) + + if args.list_tests: + list_tests() + + if args.list_tags: + list_tags() + + if args.list_allclasses: + args.list_modules = True + + if args.list_modules: + log.info('Listing all available test modules:') + testslist = get_tests(include_hidden=True) + for test in testslist: + module = test.split('oeqa.selftest.')[-1] + info = '' + if module.startswith('_'): + info = ' (hidden)' + print module + info + if args.list_allclasses: + try: + import importlib + modlib = importlib.import_module(test) + for v in vars(modlib): + t = vars(modlib)[v] + if isinstance(t, type(oeSelfTest)) and issubclass(t, oeSelfTest) and t!=oeSelfTest: + print " --", v + for method in dir(t): + if method.startswith("test_") and callable(vars(t)[method]): + print " -- --", method + + except (AttributeError, ImportError) as e: + print e + pass + + if args.run_tests or args.run_all_tests or args.run_tests_by: + if not preflight_check(): + return 1 + + if args.run_tests_by: + testslist = ts + else: + testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False) + + suite = unittest.TestSuite() + loader = unittest.TestLoader() + loader.sortTestMethodsUsing = None + runner = unittest.TextTestRunner(verbosity=2, resultclass=buildResultClass(args)) + # we need to do this here, otherwise just loading the tests + # will take 2 minutes (bitbake -e calls) + oeSelfTest.testlayer_path = get_test_layer() + for test in testslist: + log.info("Loading tests from: %s" % test) + try: + suite.addTests(loader.loadTestsFromName(test)) + except AttributeError as e: + log.error("Failed to import %s" % test) + log.error(e) + return 1 + add_include() + + if args.machine: + # Custom machine sets only weak default values (??=) for MACHINE in machine.inc + # This let test cases that require a specific MACHINE to be able to override it, using (?= or =) + log.info('Custom machine mode enabled. MACHINE set to %s' % args.machine) + if args.machine == 'random': + os.environ['CUSTOMMACHINE'] = 'random' + result = runner.run(suite) + else: # all + machines = get_available_machines() + for m in machines: + log.info('Run tests with custom MACHINE set to: %s' % m) + os.environ['CUSTOMMACHINE'] = m + result = runner.run(suite) + else: + result = runner.run(suite) + + log.info("Finished") + + if result.wasSuccessful(): + return 0 + else: + return 1 + +def buildResultClass(args): + """Build a Result Class to use in the testcase execution""" + import site + + class StampedResult(unittest.TextTestResult): + """ + Custom TestResult that prints the time when a test starts. As oe-selftest + can take a long time (ie a few hours) to run, timestamps help us understand + what tests are taking a long time to execute. + If coverage is required, this class executes the coverage setup and reporting. + """ + def startTest(self, test): + import time + self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ") + super(StampedResult, self).startTest(test) + + def startTestRun(self): + """ Setup coverage before running any testcase """ + + # variable holding the coverage configuration file allowing subprocess to be measured + self.coveragepth = None + + # indicates the system if coverage is currently installed + self.coverage_installed = True + + if args.coverage or args.coverage_source or args.coverage_include or args.coverage_omit: + try: + # check if user can do coverage + import coverage + except: + log.warn("python coverage is not installed. More info on https://pypi.python.org/pypi/coverage") + self.coverage_installed = False + + if self.coverage_installed: + log.info("Coverage is enabled") + + # In case the user has not set the variable COVERAGE_PROCESS_START, + # create a default one and export it. The COVERAGE_PROCESS_START + # value indicates where the coverage configuration file resides + # More info on https://pypi.python.org/pypi/coverage + if not os.environ.get('COVERAGE_PROCESS_START'): + os.environ['COVERAGE_PROCESS_START'] = coverage_setup(args.coverage_source, args.coverage_include, args.coverage_omit) + + # Use default site.USER_SITE and write corresponding config file + site.ENABLE_USER_SITE = True + if not os.path.exists(site.USER_SITE): + os.makedirs(site.USER_SITE) + self.coveragepth = os.path.join(site.USER_SITE, "coverage.pth") + with open(self.coveragepth, 'w') as cps: + cps.write('import sys,site; sys.path.extend(site.getsitepackages()); import coverage; coverage.process_startup();') + + def stopTestRun(self): + """ Report coverage data after the testcases are run """ + + if args.coverage or args.coverage_source or args.coverage_include or args.coverage_omit: + if self.coverage_installed: + with open(os.environ['COVERAGE_PROCESS_START']) as ccf: + log.info("Coverage configuration file (%s)" % os.environ.get('COVERAGE_PROCESS_START')) + log.info("===========================") + log.info("\n%s" % "".join(ccf.readlines())) + + log.info("Coverage Report") + log.info("===============") + try: + coverage_report() + finally: + # remove the pth file + try: + os.remove(self.coveragepth) + except OSError: + log.warn("Expected temporal file from coverage is missing, ignoring removal.") + + return StampedResult + + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + finally: + remove_include() + remove_inc_files() + sys.exit(ret) diff --git a/scripts/oe-setup-builddir b/scripts/oe-setup-builddir new file mode 100755 index 0000000..459521c --- /dev/null +++ b/scripts/oe-setup-builddir @@ -0,0 +1,128 @@ +#!/bin/sh + +# OE Build Environment Setup Script +# +# Copyright (C) 2006-2011 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +if [ -z "$BUILDDIR" ]; then + echo >&2 "Error: The build directory (BUILDDIR) must be set!" + exit 1 +fi +echo "BUILDDIR=$BUILDDIR" + +mkdir -p "$BUILDDIR/conf" + +if [ ! -d "$BUILDDIR" ]; then + echo >&2 "Error: The builddir ($BUILDDIR) does not exist!" + exit 1 +fi + +if [ ! -w "$BUILDDIR" ]; then + echo >&2 "Error: Cannot write to $BUILDDIR, perhaps try sourcing with a writable path? i.e. . oe-init-build-env ~/my-build" + exit 1 +fi + +# Attempting removal of sticky,setuid bits from BUILDDIR, BUILDDIR/conf +chmod -st "$BUILDDIR" 2>/dev/null || echo "WARNING: unable to chmod $BUILDDIR" +chmod -st "$BUILDDIR/conf" 2>/dev/null || echo "WARNING: unable to chmod $BUILDDIR/conf" + +cd "$BUILDDIR" + +if [ -f "$BUILDDIR/conf/templateconf.cfg" ]; then + TEMPLATECONF=$(cat "$BUILDDIR/conf/templateconf.cfg") +fi + +. $OEROOT/.templateconf +pwd +echo "TEMPLATECONF=$TEMPLATECONF" + +if [ ! -f "$BUILDDIR/conf/templateconf.cfg" ]; then + echo "$TEMPLATECONF" >"$BUILDDIR/conf/templateconf.cfg" +fi + +# +# $TEMPLATECONF can point to a directory for the template local.conf & bblayers.conf +# +if [ -n "$TEMPLATECONF" ]; then + if [ ! -d "$TEMPLATECONF" ]; then + # Allow TEMPLATECONF=meta-xyz/conf as a shortcut + if [ -d "$OEROOT/$TEMPLATECONF" ]; then + TEMPLATECONF="$OEROOT/$TEMPLATECONF" + fi + if [ ! -d "$TEMPLATECONF" ]; then + echo >&2 "Error: '$TEMPLATECONF' must be a directory containing local.conf & bblayers.conf" + exit 1 + fi + fi + OECORELAYERCONF="$TEMPLATECONF/bblayers.conf.sample" + OECORELOCALCONF="$TEMPLATECONF/local.conf.sample" + OECORENOTESCONF="$TEMPLATECONF/conf-notes.txt" +fi + +unset SHOWYPDOC +if [ -z "$OECORELOCALCONF" ]; then + OECORELOCALCONF="$OEROOT/meta/conf/local.conf.sample" +fi +if [ ! -r "$BUILDDIR/../conf/local.conf" ]; then + cat <<EOM +You have no conf/local.conf file. Run setup.sh. + +EOM + SHOWYPDOC=yes +fi + +if [ -z "$OECORELAYERCONF" ]; then + OECORELAYERCONF="$OEROOT/meta/conf/bblayers.conf.sample" +fi +if [ ! -r "$BUILDDIR/../conf/bblayers.conf" ]; then + cat <<EOM +You have no conf/local.conf file. Run setup.sh. + +EOM + SHOWYPDOC=yes +fi + +# Prevent disturbing a new GIT clone in same console +unset OECORELOCALCONF +unset OECORELAYERCONF + +# Ending the first-time run message. Show the YP Documentation banner. +if [ ! -z "$SHOWYPDOC" ]; then + cat <<EOM +The Yocto Project has extensive documentation about OE including a reference +manual which can be found at: + http://yoctoproject.org/documentation + +For more information about OpenEmbedded see their website: + http://www.openembedded.org/ + +EOM +# unset SHOWYPDOC +fi + +cat <<EOM + +### Shell environment set up for builds. ### + +You can now run 'bitbake <target>' + +EOM +if [ -z "$OECORENOTESCONF" ]; then + OECORENOTESCONF="$OEROOT/meta/conf/conf-notes.txt" +fi +[ ! -r "$OECORENOTESCONF" ] || cat $OECORENOTESCONF +unset OECORENOTESCONF diff --git a/scripts/oe-setup-rpmrepo b/scripts/oe-setup-rpmrepo new file mode 100755 index 0000000..917b98b --- /dev/null +++ b/scripts/oe-setup-rpmrepo @@ -0,0 +1,97 @@ +#!/bin/bash +# +# This utility setup the necessary metadata for an rpm repo +# +# Copyright (c) 2011 Intel Corp. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +# Don't use TMPDIR from the external environment, it may be a distro +# variable pointing to /tmp (e.g. within X on OpenSUSE) +# Instead, use OE_TMPDIR for passing this in externally. +TMPDIR="$OE_TMPDIR" + +function usage() { + echo "Usage: $0 <rpm-dir>" + echo " <rpm-dir>: default is $TMPDIR/deploy/rpm" +} + +if [ $# -gt 1 ]; then + usage + exit 1 +fi + +setup_tmpdir() { + if [ -z "$TMPDIR" ]; then + # Try to get TMPDIR from bitbake + type -P bitbake &>/dev/null || { + echo "In order for this script to dynamically infer paths"; + echo "to kernels or filesystem images, you either need"; + echo "bitbake in your PATH or to source oe-init-build-env"; + echo "before running this script" >&2; + exit 1; } + + # We have bitbake in PATH, get TMPDIR from bitbake + TMPDIR=`bitbake -e | grep ^TMPDIR=\" | cut -d '=' -f2 | cut -d '"' -f2` + if [ -z "$TMPDIR" ]; then + echo "Error: this script needs to be run from your build directory," + echo "or you need to explicitly set TMPDIR in your environment" + exit 1 + fi + fi +} + +setup_sysroot() { + # Toolchain installs set up $OECORE_NATIVE_SYSROOT in their + # environment script. If that variable isn't set, we're + # either in an in-tree poky scenario or the environment + # script wasn't source'd. + if [ -z "$OECORE_NATIVE_SYSROOT" ]; then + setup_tmpdir + BUILD_ARCH=`uname -m` + BUILD_OS=`uname | tr '[A-Z]' '[a-z]'` + BUILD_SYS="$BUILD_ARCH-$BUILD_OS" + + OECORE_NATIVE_SYSROOT=$TMPDIR/sysroots/$BUILD_SYS + fi +} + +setup_tmpdir +setup_sysroot + + +if [ -n "$1" ]; then + RPM_DIR="$1" +else + RPM_DIR="$TMPDIR/deploy/rpm" +fi + +if [ ! -d "$RPM_DIR" ]; then + echo "Error: rpm dir $RPM_DIR doesn't exist" + exit 1 +fi + +CREATEREPO=$OECORE_NATIVE_SYSROOT/usr/bin/createrepo +if [ ! -e "$CREATEREPO" ]; then + echo "Error: can't find createrepo binary" + echo "please run bitbake createrepo-native first" + exit 1 +fi + +export PATH=${PATH}:${OECORE_NATIVE_SYSROOT}/usr/bin + +$CREATEREPO "$RPM_DIR" + +exit 0 diff --git a/scripts/oe-trim-schemas b/scripts/oe-trim-schemas new file mode 100755 index 0000000..29fb3a1 --- /dev/null +++ b/scripts/oe-trim-schemas @@ -0,0 +1,49 @@ +#! /usr/bin/env python + +import sys +try: + import xml.etree.cElementTree as etree +except: + import xml.etree.ElementTree as etree + +def child (elem, name): + for e in elem.getchildren(): + if e.tag == name: + return e + return None + +def children (elem, name=None): + l = elem.getchildren() + if name: + l = [e for e in l if e.tag == name] + return l + +xml = etree.parse(sys.argv[1]) + +for schema in child(xml.getroot(), "schemalist").getchildren(): + e = child(schema, "short") + if e is not None: + schema.remove(e) + + e = child(schema, "long") + if e is not None: + schema.remove(e) + + for locale in children(schema, "locale"): + # One locale must exist so leave C locale... + a = locale.attrib.get("name") + if a == 'C': + continue + e = child(locale, "default") + if e is None: + schema.remove(locale) + else: + e = child(locale, "short") + if e is not None: + locale.remove(e) + e = child(locale, "long") + if e is not None: + locale.remove(e) + +xml.write(sys.stdout, "UTF-8") + diff --git a/scripts/oepydevshell-internal.py b/scripts/oepydevshell-internal.py new file mode 100755 index 0000000..f7b2e4e --- /dev/null +++ b/scripts/oepydevshell-internal.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +import os +import sys +import time +import select +import fcntl +import termios +import readline +import signal + +def nonblockingfd(fd): + fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) + +def echonocbreak(fd): + old = termios.tcgetattr(fd) + old[3] = old[3] | termios.ECHO | termios.ICANON + termios.tcsetattr(fd, termios.TCSADRAIN, old) + +def cbreaknoecho(fd): + old = termios.tcgetattr(fd) + old[3] = old[3] &~ termios.ECHO &~ termios.ICANON + termios.tcsetattr(fd, termios.TCSADRAIN, old) + +if len(sys.argv) != 3: + print("Incorrect parameters") + sys.exit(1) + +pty = open(sys.argv[1], "w+b", 0) +parent = int(sys.argv[2]) + +# Don't buffer output by line endings +sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) +sys.stdin = os.fdopen(sys.stdin.fileno(), 'r', 0) +nonblockingfd(pty) +nonblockingfd(sys.stdin) + + +histfile = os.path.expanduser("~/.oedevpyshell-history") +readline.parse_and_bind("tab: complete") +try: + readline.read_history_file(histfile) +except IOError: + pass + +try: + + i = "" + o = "" + # Need cbreak/noecho whilst in select so we trigger on any keypress + cbreaknoecho(sys.stdin.fileno()) + # Send our PID to the other end so they can kill us. + pty.write(str(os.getpid()) + "\n") + while True: + try: + writers = [] + if i: + writers.append(sys.stdout) + (ready, _, _) = select.select([pty, sys.stdin], writers , [], 0) + try: + if pty in ready: + i = i + pty.read() + if i: + # Write a page at a time to avoid overflowing output + # d.keys() is a good way to do that + sys.stdout.write(i[:4096]) + i = i[4096:] + if sys.stdin in ready: + echonocbreak(sys.stdin.fileno()) + o = raw_input() + cbreaknoecho(sys.stdin.fileno()) + pty.write(o + "\n") + except (IOError, OSError) as e: + if e.errno == 11: + continue + if e.errno == 5: + sys.exit(0) + raise + except EOFError: + sys.exit(0) + except KeyboardInterrupt: + os.kill(parent, signal.SIGINT) + +except SystemExit: + pass +except Exception as e: + import traceback + print("Exception in oepydehshell-internal: " + str(e)) + traceback.print_exc() + time.sleep(5) +finally: + readline.write_history_file(histfile) diff --git a/scripts/opkg-query-helper.py b/scripts/opkg-query-helper.py new file mode 100755 index 0000000..2fb1a78 --- /dev/null +++ b/scripts/opkg-query-helper.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +# OpenEmbedded opkg query helper utility +# +# Written by: Paul Eggleton <paul.eggleton@linux.intel.com> +# +# Copyright 2012 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# + + +import sys +import fileinput +import re + +archmode = False +filemode = False +vermode = False + +args = [] +for arg in sys.argv[1:]: + if arg == '-a': + archmode = True + elif arg == '-f': + filemode = True + elif arg == '-v': + vermode = True + else: + args.append(arg) + +# Regex for removing version specs after dependency items +verregex = re.compile(' \([=<>]* [^ )]*\)') + +pkg = "" +ver = "" +for line in fileinput.input(args): + line = line.rstrip() + if ': ' in line: + if line.startswith("Package:"): + pkg = line.split(": ")[1] + ver = "" + else: + if archmode: + if line.startswith("Architecture:"): + arch = line.split(": ")[1] + print("%s %s" % (pkg,arch)) + elif filemode: + if line.startswith("Version:"): + ver = line.split(": ")[1] + elif line.startswith("Architecture:"): + arch = line.split(": ")[1] + print("%s %s_%s_%s.ipk %s" % (pkg,pkg,ver,arch,arch)) + elif vermode: + if line.startswith("Version:"): + ver = line.split(": ")[1] + elif line.startswith("Architecture:"): + arch = line.split(": ")[1] + print("%s %s %s" % (pkg,arch,ver)) + else: + if line.startswith("Depends:"): + depval = line.split(": ")[1] + deps = depval.split(", ") + for dep in deps: + dep = verregex.sub('', dep) + print("%s|%s" % (pkg,dep)) + elif line.startswith("Recommends:"): + recval = line.split(": ")[1] + recs = recval.split(", ") + for rec in recs: + rec = verregex.sub('', rec) + print("%s|%s [REC]" % (pkg, rec)) + diff --git a/scripts/postinst-intercepts/postinst_intercept b/scripts/postinst-intercepts/postinst_intercept new file mode 100755 index 0000000..b18e806 --- /dev/null +++ b/scripts/postinst-intercepts/postinst_intercept @@ -0,0 +1,56 @@ +#!/bin/sh +# +# This script is called from inside postinstall scriptlets at do_rootfs time. It +# actually adds, at the end, the list of packages for which the intercept script +# is valid. Also, if one wants to pass any variables to the intercept script from +# the postinstall itself, they will be added immediately after the shebang line. +# +# Usage: postinst_intercept <intercept_script_name> <package_name> <mlprefix=...> <var1=...> ... <varN=...> +# * intercept_script_name - the name of the intercept script we want to change; +# * package_name - add the package_name to list of packages the intercept script +# is used for; +# * mlprefix=... - this one is needed in order to have separate hooks for multilib. +# * var1=... - var1 will have the value we provide in the intercept script. This +# is useful when we want to pass on variables like ${libdir} to +# the intercept script; +# +[ $# -lt 3 ] && exit 1 + +intercept_script=$INTERCEPT_DIR/$1 && shift +package_name=$1 && shift +mlprefix=$(echo $1 |sed -ne "s/^mlprefix=\(.*\)-/\1/p") && shift + +# if the hook we want to install does not exist, then there's nothing we can do +[ -f "$intercept_script" ] || exit 1 + +# if the postinstall wanting to install the hook belongs to a multilib package, +# then we'd better have a separate hook for this because the default ${libdir} and +# ${base_libdir} will point to the wrong locations +if [ -n "$mlprefix" ]; then + ml_intercept_script=$intercept_script-$mlprefix + # if the multilib hook does not exist, create it from the default one + if [ ! -f "$ml_intercept_script" ]; then + cp $intercept_script $ml_intercept_script + + # clear the ##PKGS: line and the already set variables + [ -x "$ml_intercept_script" ] && sed -i -e "2,$(($#+1)) {/.*/d}" -e "/^##PKGS: .*/d" $ml_intercept_script + fi + + intercept_script=$ml_intercept_script +fi + +chmod +x "$intercept_script" + +pkgs_line=$(grep "##PKGS:" $intercept_script) +if [ -n "$pkgs_line" ]; then + # line exists, add this package to the list only if it's not already there + if [ -z "$(echo "$pkgs_line" | grep " $package_name ")" ]; then + sed -i -e "s/##PKGS:.*/\0${package_name} /" $intercept_script + fi +else + for var in "$@"; do + sed -i -e "\%^#\!/bin/.*sh%a $var" $intercept_script + done + echo "##PKGS: ${package_name} " >> $intercept_script +fi + diff --git a/scripts/postinst-intercepts/update_font_cache b/scripts/postinst-intercepts/update_font_cache new file mode 100644 index 0000000..bf65e19 --- /dev/null +++ b/scripts/postinst-intercepts/update_font_cache @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +PSEUDO_UNLOAD=1 qemuwrapper -L $D -E LD_LIBRARY_PATH=$D/${libdir}:$D/${base_libdir} \ + -E ${fontconfigcacheenv} $D${bindir}/fc-cache --sysroot=$D --system-only ${fontconfigcacheparams} +chown -R root:root $D${fontconfigcachedir} diff --git a/scripts/postinst-intercepts/update_gio_module_cache b/scripts/postinst-intercepts/update_gio_module_cache new file mode 100644 index 0000000..fe46809 --- /dev/null +++ b/scripts/postinst-intercepts/update_gio_module_cache @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +PSEUDO_UNLOAD=1 qemuwrapper -L $D -E LD_LIBRARY_PATH=$D${libdir}:$D${base_libdir} \ + $D${libexecdir}/${binprefix}gio-querymodules $D${libdir}/gio/modules/ + diff --git a/scripts/postinst-intercepts/update_icon_cache b/scripts/postinst-intercepts/update_icon_cache new file mode 100644 index 0000000..9cf2a72 --- /dev/null +++ b/scripts/postinst-intercepts/update_icon_cache @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +# update native pixbuf loaders +$STAGING_DIR_NATIVE/${libdir_native}/gdk-pixbuf-2.0/gdk-pixbuf-query-loaders --update-cache + +for icondir in $D/usr/share/icons/*/ ; do + if [ -d $icondir ] ; then + gtk-update-icon-cache -fqt $icondir + fi +done + diff --git a/scripts/postinst-intercepts/update_pixbuf_cache b/scripts/postinst-intercepts/update_pixbuf_cache new file mode 100644 index 0000000..5d44075 --- /dev/null +++ b/scripts/postinst-intercepts/update_pixbuf_cache @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +export GDK_PIXBUF_MODULEDIR=$D${libdir}/gdk-pixbuf-2.0/2.10.0/loaders +export GDK_PIXBUF_FATAL_LOADER=1 + +PSEUDO_UNLOAD=1 qemuwrapper -L $D -E LD_LIBRARY_PATH=$D/${libdir}:$D/${base_libdir}\ + $D${libdir}/gdk-pixbuf-2.0/gdk-pixbuf-query-loaders \ + >$GDK_PIXBUF_MODULEDIR/../loaders.cache && \ + sed -i -e "s:$D::g" $GDK_PIXBUF_MODULEDIR/../loaders.cache diff --git a/scripts/pybootchartgui/AUTHORS b/scripts/pybootchartgui/AUTHORS new file mode 100644 index 0000000..672b7e9 --- /dev/null +++ b/scripts/pybootchartgui/AUTHORS @@ -0,0 +1,11 @@ +Michael Meeks <michael.meeks@novell.com> +Anders Norgaard <anders.norgaard@gmail.com> +Scott James Remnant <scott@ubuntu.com> +Henning Niss <henningniss@gmail.com> +Riccardo Magliocchetti <riccardo.magliocchetti@gmail.com> + +Contributors: + Brian Ewins + +Based on work by: + Ziga Mahkovec diff --git a/scripts/pybootchartgui/COPYING b/scripts/pybootchartgui/COPYING new file mode 100644 index 0000000..ed87acf --- /dev/null +++ b/scripts/pybootchartgui/COPYING @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/scripts/pybootchartgui/MAINTAINERS b/scripts/pybootchartgui/MAINTAINERS new file mode 100644 index 0000000..c65e131 --- /dev/null +++ b/scripts/pybootchartgui/MAINTAINERS @@ -0,0 +1,3 @@ +Riccardo Magliocchetti <riccardo.magliocchetti@gmail.com> +Michael Meeks <michael.meeks@novell.com> +Harald Hoyer <harald@redhat.com> diff --git a/scripts/pybootchartgui/NEWS b/scripts/pybootchartgui/NEWS new file mode 100644 index 0000000..7c5b2fc --- /dev/null +++ b/scripts/pybootchartgui/NEWS @@ -0,0 +1,204 @@ +bootchart2 0.14.5: + + pybootchartgui (Riccardo) + + Fix tests with python3 + + Fix parsing of files with non-ascii bytes + + Robustness fixes to taskstats and meminfo parsing + + More python3 fixes + +bootchart2 0.14.4: + + bootchartd + + Add relevant EXIT_PROC for GNOME3, XFCE4, openbox + (Justin Lecher, Ben Eills) + + pybootchartgui (Riccardo) + + Fix some issues in --crop-after and --annotate + + Fix pybootchartgui process_tree tests + + More python3 fixes + +bootchart2 0.14.2: + + pybootchartgui + + Fix some crashes in parsing.py (Jakub Czaplicki, Riccardo) + + speedup a bit meminfo parsing (Riccardo) + + Fix indentation for python3.2 (Riccardo) + +bootchart2 0.14.1: + + bootchartd + + Expect dmesg only if started as init (Henry Yei) + + look for bootchart_init in the environment (Henry Gebhardt) + + pybootchartgui + + Fixup some tests (Riccardo) + + Support hp smart arrays block devices (Anders Norgaard, + Brian Murray) + + Fixes for -t, -o and -f options (Mladen Kuntner, Harald, Riccardo) + +bootchart2 0.14.0: + + bootchartd + + Add ability to define custom commands + (Lucian Muresan, Peter Hjalmarsson) + + collector + + fix tmpfs mount leakage (Peter Hjalmarsson) + + pybootchartgui + + render cumulative I/O time chart (Sankar P) + + python3 compatibility fixes (Riccardo) + + Misc (Michael) + + remove confusing, obsolete setup.py + + install docs to /usr/share/ + + lot of fixes for easier packaging (Peter Hjalmarsson) + + add bootchart2, bootchartd and pybootchartgui manpages + (Francesca Ciceri, David Paleino) + +bootchart2 0.12.6: + + bootchartd + + better check for initrd (Riccardo Magliocchetti) + + code cleanup (Riccardo) + + make the list of processes we are waiting for editable + in config file by EXIT_PROC (Riccardo) + + fix parsing of cmdline for alternative init system (Riccardo) + + fixed calling init in initramfs (Harald) + + exit 0 for start, if the collector is already running (Harald) + + collector + + try harder with taskstats (Michael) + + plug some small leaks (Riccardo) + + fix missing PROC_EVENTS detection (Harald) + + pybootchartgui (Michael) + + add kernel bootchart tab to interactive gui + + report bootchart version in cli interface + + improve rendering performance + + GUI improvements + + lot of cleanups + + Makefile + + do not python compile if NO_PYTHON_COMPILE is set (Harald) + + systemd service files + + added them and install (Harald, Wulf C. Krueger) + +bootchart2 0.12.5: + + administrative snafu version; pull before pushing... + +bootchart2 0.12.4: + + bootchartd + + reduce overhead caused by pidof (Riccardo Magliocchetti) + + collector + + attempt to retry ptrace to avoid bogus ENOSYS (Michael) + + add meminfo polling (Dave Martin) + + pybootchartgui + + handle dmesg timestamps with big delta (Riccardo) + + avoid divide by zero when rendering I/O utilization (Riccardo) + + add process grouping in the cumulative chart (Riccardo) + + fix cpu time calculation in cumulative chart (Riccardo) + + get i/o statistics for flash based devices (Riccardo) + + prettier coloring for the cumulative graphs (Michael) + + fix interactive CPU rendering (Michael) + + render memory usage graph (Dave Martin) + +bootchart2 0.12.3 + + collector + + pclose after popen (Riccardo Magliocchetti (xrmx)) + + fix buffer overflow (xrmx) + + count 'processor:' in /proc/cpuinfo for ARM (Michael) + + get model name from that line too for ARM (xrmx) + + store /proc/cpuinfo in the boot-chart archive (xrmx) + + try harder to detect missing TASKSTATS (Michael) + + sanity-check invalid domain names (Michael) + + detect missing PROC_EVENTS more reliably (Michael) + + README fixes (xrmx, Michael) + + pybootchartgui + + make num_cpu parsing robust (Michael) + +bootchart2 0.12.2 + + fix pthread compile / linking bug + +bootchart2 0.12.1 + + pybootchartgui + + pylint cleanup + + handle empty traces more elegantly + + add '-t' / '--boot-time' argument (Matthew Bauer) + + collector + + now GPLv2 + + add rdinit support for very early initrd tracing + + cleanup / re-factor code into separate modules + + re-factor arg parsing, and parse remote process args + + handle missing bootchartd.conf cleanly + + move much of bootchartd from shell -> C + + drop dmesg and uname usage + + avoid rpm/dpkg with native version reporting + +bootchart2 0.12.0 (Michael Meeks) + + collector + + use netlink PROC_EVENTS to generate parentage data + + finally kills any need for 'acct' et. al. + + also removes need to poll /proc => faster + + cleanup code to K&R, 8 stop tabs. + + pybootchartgui + + consume thread parentage data + +bootchart2 0.11.4 (Michael Meeks) + + collector + + if run inside an initrd detect when /dev is writable + and remount ourselves into that. + + overflow buffers more elegantly in extremis + + dump full process path and command-line args + + calm down debugging output + + pybootchartgui + + can render logs in a directory again + + has a 'show more' option to show command-lines + +bootchart2 0.11.3 (Michael Meeks) + + add $$ display to the bootchart header + + process command-line bits + + fix collection code, and rename stream to match + + enable parsing, add check button to UI, and --show-all + command-line option + + fix parsing of directories full of files. + +bootchart2 0.11.2 (Michael Meeks) + + fix initrd sanity check to use the right proc path + + don't return a bogus error value when dumping state + + add -c to aid manual console debugging + +bootchart2 0.11.1 (Michael Meeks) + + even simpler initrd setup + + create a single directory: /lib/bootchart/tmpfs + +bootchart2 0.11 (Michael Meeks) + + bootchartd + + far, far simpler, less shell, more robustness etc. + + bootchart-collector + + remove the -p argument - we always mount proc + + requires /lib/bootchart (make install-chroot) to + be present (also in the initrd) [ with a kmsg + node included ] + + add a --probe-running mode + + ptrace re-write + + gives -much- better early-boot-time resolution + + unconditional chroot /lib/bootchart/chroot + + we mount proc there ourselves + + log extraction requires no common file-system view + + +bootchart2 0.10.1 (Kel Modderman) + + collector arg -m should mount /proc + + remove bogus vcsid code + + split collector install in Makefile + + remove bogus debug code + + accept process names containing spaces + +bootchart2 0.10.0 + + rendering (Anders Norgaard) + + fix for unknown exceptions + + interactive UI (Michael) + + much faster rendering by manual clipping + + horizontal scaling + + remove annoying page-up/down bindings + + initrd portability & fixes (Federic Crozat) + + port to Mandriva + + improved process waiting + + inittab commenting fix + + improved initrd detection / jail tagging + + fix for un-detectable accton behaviour change + + implement a built-in usleep to help initrd deps (Michael) + +bootchart2 0.0.9 + + fix initrd bug + +bootchart2 0.0.8 + + add a filename string to the window title in interactive mode + + add a NEWS file diff --git a/scripts/pybootchartgui/README.pybootchart b/scripts/pybootchartgui/README.pybootchart new file mode 100644 index 0000000..8642e64 --- /dev/null +++ b/scripts/pybootchartgui/README.pybootchart @@ -0,0 +1,37 @@ + PYBOOTCHARTGUI + ---------------- + +pybootchartgui is a tool (now included as part of bootchart2) for +visualization and analysis of the GNU/Linux boot process. It renders +the output of the boot-logger tool bootchart (see +http://www.bootchart.org/) to either the screen or files of various +formats. Bootchart collects information about the processes, their +dependencies, and resource consumption during boot of a GNU/Linux +system. The pybootchartgui tools visualizes the process tree and +overall resource utilization. + +pybootchartgui is a port of the visualization part of bootchart from +Java to Python and Cairo. + +Adapted from the bootchart-documentation: + + The CPU and disk statistics are used to render stacked area and line + charts. The process information is used to create a Gantt chart + showing process dependency, states and CPU usage. + + A typical boot sequence consists of several hundred processes. Since + it is difficult to visualize such amount of data in a comprehensible + way, tree pruning is utilized. Idle background processes and + short-lived processes are removed. Similar processes running in + parallel are also merged together. + + Finally, the performance and dependency charts are rendered as a + single image to either the screen or in PNG, PDF or SVG format. + + +To get help for pybootchartgui, run + +$ pybootchartgui --help + +This code was originally hosted at: + http://code.google.com/p/pybootchartgui/ diff --git a/scripts/pybootchartgui/pybootchartgui.py b/scripts/pybootchartgui/pybootchartgui.py new file mode 100755 index 0000000..7ce1a5b --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# +# This file is part of pybootchartgui. + +# pybootchartgui is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# pybootchartgui is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with pybootchartgui. If not, see <http://www.gnu.org/licenses/>. + + +import sys +from pybootchartgui.main import main + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/pybootchartgui/pybootchartgui/__init__.py b/scripts/pybootchartgui/pybootchartgui/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/__init__.py diff --git a/scripts/pybootchartgui/pybootchartgui/batch.py b/scripts/pybootchartgui/pybootchartgui/batch.py new file mode 100644 index 0000000..05c714e --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/batch.py @@ -0,0 +1,46 @@ +# This file is part of pybootchartgui. + +# pybootchartgui is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# pybootchartgui is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with pybootchartgui. If not, see <http://www.gnu.org/licenses/>. + +import cairo +from . import draw +from .draw import RenderOptions + +def render(writer, trace, app_options, filename): + handlers = { + "png": (lambda w, h: cairo.ImageSurface(cairo.FORMAT_ARGB32, w, h), \ + lambda sfc: sfc.write_to_png(filename)), + "pdf": (lambda w, h: cairo.PDFSurface(filename, w, h), lambda sfc: 0), + "svg": (lambda w, h: cairo.SVGSurface(filename, w, h), lambda sfc: 0) + } + + if app_options.format is None: + fmt = filename.rsplit('.', 1)[1] + else: + fmt = app_options.format + + if not (fmt in handlers): + writer.error ("Unknown format '%s'." % fmt) + return 10 + + make_surface, write_surface = handlers[fmt] + options = RenderOptions (app_options) + (w, h) = draw.extents (options, 1.0, trace) + w = max (w, draw.MIN_IMG_W) + surface = make_surface (w, h) + ctx = cairo.Context (surface) + draw.render (ctx, options, 1.0, trace) + write_surface (surface) + writer.status ("bootchart written to '%s'" % filename) + diff --git a/scripts/pybootchartgui/pybootchartgui/draw.py b/scripts/pybootchartgui/pybootchartgui/draw.py new file mode 100644 index 0000000..8c574be --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/draw.py @@ -0,0 +1,894 @@ +# This file is part of pybootchartgui. + +# pybootchartgui is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# pybootchartgui is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with pybootchartgui. If not, see <http://www.gnu.org/licenses/>. + + +import cairo +import math +import re +import random +import colorsys +from operator import itemgetter + +class RenderOptions: + + def __init__(self, app_options): + # should we render a cumulative CPU time chart + self.cumulative = True + self.charts = True + self.kernel_only = False + self.app_options = app_options + + def proc_tree (self, trace): + if self.kernel_only: + return trace.kernel_tree + else: + return trace.proc_tree + +# Process tree background color. +BACK_COLOR = (1.0, 1.0, 1.0, 1.0) + +WHITE = (1.0, 1.0, 1.0, 1.0) +# Process tree border color. +BORDER_COLOR = (0.63, 0.63, 0.63, 1.0) +# Second tick line color. +TICK_COLOR = (0.92, 0.92, 0.92, 1.0) +# 5-second tick line color. +TICK_COLOR_BOLD = (0.86, 0.86, 0.86, 1.0) +# Annotation colour +ANNOTATION_COLOR = (0.63, 0.0, 0.0, 0.5) +# Text color. +TEXT_COLOR = (0.0, 0.0, 0.0, 1.0) + +# Font family +FONT_NAME = "Bitstream Vera Sans" +# Title text font. +TITLE_FONT_SIZE = 18 +# Default text font. +TEXT_FONT_SIZE = 12 +# Axis label font. +AXIS_FONT_SIZE = 11 +# Legend font. +LEGEND_FONT_SIZE = 12 + +# CPU load chart color. +CPU_COLOR = (0.40, 0.55, 0.70, 1.0) +# IO wait chart color. +IO_COLOR = (0.76, 0.48, 0.48, 0.5) +# Disk throughput color. +DISK_TPUT_COLOR = (0.20, 0.71, 0.20, 1.0) +# CPU load chart color. +FILE_OPEN_COLOR = (0.20, 0.71, 0.71, 1.0) +# Mem cached color +MEM_CACHED_COLOR = CPU_COLOR +# Mem used color +MEM_USED_COLOR = IO_COLOR +# Buffers color +MEM_BUFFERS_COLOR = (0.4, 0.4, 0.4, 0.3) +# Swap color +MEM_SWAP_COLOR = DISK_TPUT_COLOR + +# Process border color. +PROC_BORDER_COLOR = (0.71, 0.71, 0.71, 1.0) +# Waiting process color. +PROC_COLOR_D = (0.76, 0.48, 0.48, 0.5) +# Running process color. +PROC_COLOR_R = CPU_COLOR +# Sleeping process color. +PROC_COLOR_S = (0.94, 0.94, 0.94, 1.0) +# Stopped process color. +PROC_COLOR_T = (0.94, 0.50, 0.50, 1.0) +# Zombie process color. +PROC_COLOR_Z = (0.71, 0.71, 0.71, 1.0) +# Dead process color. +PROC_COLOR_X = (0.71, 0.71, 0.71, 0.125) +# Paging process color. +PROC_COLOR_W = (0.71, 0.71, 0.71, 0.125) + +# Process label color. +PROC_TEXT_COLOR = (0.19, 0.19, 0.19, 1.0) +# Process label font. +PROC_TEXT_FONT_SIZE = 12 + +# Signature color. +SIG_COLOR = (0.0, 0.0, 0.0, 0.3125) +# Signature font. +SIG_FONT_SIZE = 14 +# Signature text. +SIGNATURE = "http://github.com/mmeeks/bootchart" + +# Process dependency line color. +DEP_COLOR = (0.75, 0.75, 0.75, 1.0) +# Process dependency line stroke. +DEP_STROKE = 1.0 + +# Process description date format. +DESC_TIME_FORMAT = "mm:ss.SSS" + +# Cumulative coloring bits +HSV_MAX_MOD = 31 +HSV_STEP = 7 + +# Configure task color +TASK_COLOR_CONFIGURE = (1.0, 1.0, 0.00, 1.0) +# Compile task color. +TASK_COLOR_COMPILE = (0.0, 1.00, 0.00, 1.0) +# Install task color +TASK_COLOR_INSTALL = (1.0, 0.00, 1.00, 1.0) +# Sysroot task color +TASK_COLOR_SYSROOT = (0.0, 0.00, 1.00, 1.0) +# Package task color +TASK_COLOR_PACKAGE = (0.0, 1.00, 1.00, 1.0) +# Package Write RPM/DEB/IPK task color +TASK_COLOR_PACKAGE_WRITE = (0.0, 0.50, 0.50, 1.0) + +# Process states +STATE_UNDEFINED = 0 +STATE_RUNNING = 1 +STATE_SLEEPING = 2 +STATE_WAITING = 3 +STATE_STOPPED = 4 +STATE_ZOMBIE = 5 + +STATE_COLORS = [(0, 0, 0, 0), PROC_COLOR_R, PROC_COLOR_S, PROC_COLOR_D, \ + PROC_COLOR_T, PROC_COLOR_Z, PROC_COLOR_X, PROC_COLOR_W] + +# CumulativeStats Types +STAT_TYPE_CPU = 0 +STAT_TYPE_IO = 1 + +# Convert ps process state to an int +def get_proc_state(flag): + return "RSDTZXW".find(flag) + 1 + +def draw_text(ctx, text, color, x, y): + ctx.set_source_rgba(*color) + ctx.move_to(x, y) + ctx.show_text(text) + +def draw_fill_rect(ctx, color, rect): + ctx.set_source_rgba(*color) + ctx.rectangle(*rect) + ctx.fill() + +def draw_rect(ctx, color, rect): + ctx.set_source_rgba(*color) + ctx.rectangle(*rect) + ctx.stroke() + +def draw_legend_box(ctx, label, fill_color, x, y, s): + draw_fill_rect(ctx, fill_color, (x, y - s, s, s)) + draw_rect(ctx, PROC_BORDER_COLOR, (x, y - s, s, s)) + draw_text(ctx, label, TEXT_COLOR, x + s + 5, y) + +def draw_legend_line(ctx, label, fill_color, x, y, s): + draw_fill_rect(ctx, fill_color, (x, y - s/2, s + 1, 3)) + ctx.arc(x + (s + 1)/2.0, y - (s - 3)/2.0, 2.5, 0, 2.0 * math.pi) + ctx.fill() + draw_text(ctx, label, TEXT_COLOR, x + s + 5, y) + +def draw_label_in_box(ctx, color, label, x, y, w, maxx): + label_w = ctx.text_extents(label)[2] + label_x = x + w / 2 - label_w / 2 + if label_w + 10 > w: + label_x = x + w + 5 + if label_x + label_w > maxx: + label_x = x - label_w - 5 + draw_text(ctx, label, color, label_x, y) + +def draw_sec_labels(ctx, options, rect, sec_w, nsecs): + ctx.set_font_size(AXIS_FONT_SIZE) + prev_x = 0 + for i in range(0, rect[2] + 1, sec_w): + if ((i / sec_w) % nsecs == 0) : + if options.app_options.as_minutes : + label = "%.1f" % (i / sec_w / 60.0) + else : + label = "%d" % (i / sec_w) + label_w = ctx.text_extents(label)[2] + x = rect[0] + i - label_w/2 + if x >= prev_x: + draw_text(ctx, label, TEXT_COLOR, x, rect[1] - 2) + prev_x = x + label_w + +def draw_box_ticks(ctx, rect, sec_w): + draw_rect(ctx, BORDER_COLOR, tuple(rect)) + + ctx.set_line_cap(cairo.LINE_CAP_SQUARE) + + for i in range(sec_w, rect[2] + 1, sec_w): + if ((i / sec_w) % 10 == 0) : + ctx.set_line_width(1.5) + elif sec_w < 5 : + continue + else : + ctx.set_line_width(1.0) + if ((i / sec_w) % 30 == 0) : + ctx.set_source_rgba(*TICK_COLOR_BOLD) + else : + ctx.set_source_rgba(*TICK_COLOR) + ctx.move_to(rect[0] + i, rect[1] + 1) + ctx.line_to(rect[0] + i, rect[1] + rect[3] - 1) + ctx.stroke() + ctx.set_line_width(1.0) + + ctx.set_line_cap(cairo.LINE_CAP_BUTT) + +def draw_annotations(ctx, proc_tree, times, rect): + ctx.set_line_cap(cairo.LINE_CAP_SQUARE) + ctx.set_source_rgba(*ANNOTATION_COLOR) + ctx.set_dash([4, 4]) + + for time in times: + if time is not None: + x = ((time - proc_tree.start_time) * rect[2] / proc_tree.duration) + + ctx.move_to(rect[0] + x, rect[1] + 1) + ctx.line_to(rect[0] + x, rect[1] + rect[3] - 1) + ctx.stroke() + + ctx.set_line_cap(cairo.LINE_CAP_BUTT) + ctx.set_dash([]) + +def draw_chart(ctx, color, fill, chart_bounds, data, proc_tree, data_range): + ctx.set_line_width(0.5) + x_shift = proc_tree.start_time + + def transform_point_coords(point, x_base, y_base, \ + xscale, yscale, x_trans, y_trans): + x = (point[0] - x_base) * xscale + x_trans + y = (point[1] - y_base) * -yscale + y_trans + chart_bounds[3] + return x, y + + max_x = max (x for (x, y) in data) + max_y = max (y for (x, y) in data) + # avoid divide by zero + if max_y == 0: + max_y = 1.0 + xscale = float (chart_bounds[2]) / max_x + # If data_range is given, scale the chart so that the value range in + # data_range matches the chart bounds exactly. + # Otherwise, scale so that the actual data matches the chart bounds. + if data_range: + yscale = float(chart_bounds[3]) / (data_range[1] - data_range[0]) + ybase = data_range[0] + else: + yscale = float(chart_bounds[3]) / max_y + ybase = 0 + + first = transform_point_coords (data[0], x_shift, ybase, xscale, yscale, \ + chart_bounds[0], chart_bounds[1]) + last = transform_point_coords (data[-1], x_shift, ybase, xscale, yscale, \ + chart_bounds[0], chart_bounds[1]) + + ctx.set_source_rgba(*color) + ctx.move_to(*first) + for point in data: + x, y = transform_point_coords (point, x_shift, ybase, xscale, yscale, \ + chart_bounds[0], chart_bounds[1]) + ctx.line_to(x, y) + if fill: + ctx.stroke_preserve() + ctx.line_to(last[0], chart_bounds[1]+chart_bounds[3]) + ctx.line_to(first[0], chart_bounds[1]+chart_bounds[3]) + ctx.line_to(first[0], first[1]) + ctx.fill() + else: + ctx.stroke() + ctx.set_line_width(1.0) + +bar_h = 55 +meminfo_bar_h = 2 * bar_h +header_h = 60 +# offsets +off_x, off_y = 220, 10 +sec_w_base = 1 # the width of a second +proc_h = 16 # the height of a process +leg_s = 10 +MIN_IMG_W = 800 +CUML_HEIGHT = 2000 # Increased value to accomodate CPU and I/O Graphs +OPTIONS = None + +def extents(options, xscale, trace): + start = min(trace.start.keys()) + end = start + + processes = 0 + for proc in trace.processes: + if not options.app_options.show_all and \ + trace.processes[proc][1] - trace.processes[proc][0] < options.app_options.mintime: + continue + + if trace.processes[proc][1] > end: + end = trace.processes[proc][1] + processes += 1 + + if trace.min is not None and trace.max is not None: + start = trace.min + end = trace.max + + w = int ((end - start) * sec_w_base * xscale) + 2 * off_x + h = proc_h * processes + header_h + 2 * off_y + + return (w, h) + +def clip_visible(clip, rect): + xmax = max (clip[0], rect[0]) + ymax = max (clip[1], rect[1]) + xmin = min (clip[0] + clip[2], rect[0] + rect[2]) + ymin = min (clip[1] + clip[3], rect[1] + rect[3]) + return (xmin > xmax and ymin > ymax) + +def render_charts(ctx, options, clip, trace, curr_y, w, h, sec_w): + proc_tree = options.proc_tree(trace) + + # render bar legend + ctx.set_font_size(LEGEND_FONT_SIZE) + + draw_legend_box(ctx, "CPU (user+sys)", CPU_COLOR, off_x, curr_y+20, leg_s) + draw_legend_box(ctx, "I/O (wait)", IO_COLOR, off_x + 120, curr_y+20, leg_s) + + # render I/O wait + chart_rect = (off_x, curr_y+30, w, bar_h) + if clip_visible (clip, chart_rect): + draw_box_ticks (ctx, chart_rect, sec_w) + draw_annotations (ctx, proc_tree, trace.times, chart_rect) + draw_chart (ctx, IO_COLOR, True, chart_rect, \ + [(sample.time, sample.user + sample.sys + sample.io) for sample in trace.cpu_stats], \ + proc_tree, None) + # render CPU load + draw_chart (ctx, CPU_COLOR, True, chart_rect, \ + [(sample.time, sample.user + sample.sys) for sample in trace.cpu_stats], \ + proc_tree, None) + + curr_y = curr_y + 30 + bar_h + + # render second chart + draw_legend_line(ctx, "Disk throughput", DISK_TPUT_COLOR, off_x, curr_y+20, leg_s) + draw_legend_box(ctx, "Disk utilization", IO_COLOR, off_x + 120, curr_y+20, leg_s) + + # render I/O utilization + chart_rect = (off_x, curr_y+30, w, bar_h) + if clip_visible (clip, chart_rect): + draw_box_ticks (ctx, chart_rect, sec_w) + draw_annotations (ctx, proc_tree, trace.times, chart_rect) + draw_chart (ctx, IO_COLOR, True, chart_rect, \ + [(sample.time, sample.util) for sample in trace.disk_stats], \ + proc_tree, None) + + # render disk throughput + max_sample = max (trace.disk_stats, key = lambda s: s.tput) + if clip_visible (clip, chart_rect): + draw_chart (ctx, DISK_TPUT_COLOR, False, chart_rect, \ + [(sample.time, sample.tput) for sample in trace.disk_stats], \ + proc_tree, None) + + pos_x = off_x + ((max_sample.time - proc_tree.start_time) * w / proc_tree.duration) + + shift_x, shift_y = -20, 20 + if (pos_x < off_x + 245): + shift_x, shift_y = 5, 40 + + label = "%dMB/s" % round ((max_sample.tput) / 1024.0) + draw_text (ctx, label, DISK_TPUT_COLOR, pos_x + shift_x, curr_y + shift_y) + + curr_y = curr_y + 30 + bar_h + + # render mem usage + chart_rect = (off_x, curr_y+30, w, meminfo_bar_h) + mem_stats = trace.mem_stats + if mem_stats and clip_visible (clip, chart_rect): + mem_scale = max(sample.records['MemTotal'] - sample.records['MemFree'] for sample in mem_stats) + draw_legend_box(ctx, "Mem cached (scale: %u MiB)" % (float(mem_scale) / 1024), MEM_CACHED_COLOR, off_x, curr_y+20, leg_s) + draw_legend_box(ctx, "Used", MEM_USED_COLOR, off_x + 240, curr_y+20, leg_s) + draw_legend_box(ctx, "Buffers", MEM_BUFFERS_COLOR, off_x + 360, curr_y+20, leg_s) + draw_legend_line(ctx, "Swap (scale: %u MiB)" % max([(sample.records['SwapTotal'] - sample.records['SwapFree'])/1024 for sample in mem_stats]), \ + MEM_SWAP_COLOR, off_x + 480, curr_y+20, leg_s) + draw_box_ticks(ctx, chart_rect, sec_w) + draw_annotations(ctx, proc_tree, trace.times, chart_rect) + draw_chart(ctx, MEM_BUFFERS_COLOR, True, chart_rect, \ + [(sample.time, sample.records['MemTotal'] - sample.records['MemFree']) for sample in trace.mem_stats], \ + proc_tree, [0, mem_scale]) + draw_chart(ctx, MEM_USED_COLOR, True, chart_rect, \ + [(sample.time, sample.records['MemTotal'] - sample.records['MemFree'] - sample.records['Buffers']) for sample in mem_stats], \ + proc_tree, [0, mem_scale]) + draw_chart(ctx, MEM_CACHED_COLOR, True, chart_rect, \ + [(sample.time, sample.records['Cached']) for sample in mem_stats], \ + proc_tree, [0, mem_scale]) + draw_chart(ctx, MEM_SWAP_COLOR, False, chart_rect, \ + [(sample.time, float(sample.records['SwapTotal'] - sample.records['SwapFree'])) for sample in mem_stats], \ + proc_tree, None) + + curr_y = curr_y + meminfo_bar_h + + return curr_y + +def render_processes_chart(ctx, options, trace, curr_y, w, h, sec_w): + chart_rect = [off_x, curr_y+header_h, w, h - 2 * off_y - (curr_y+header_h) + proc_h] + + draw_legend_box (ctx, "Configure", \ + TASK_COLOR_CONFIGURE, off_x , curr_y + 45, leg_s) + draw_legend_box (ctx, "Compile", \ + TASK_COLOR_COMPILE, off_x+120, curr_y + 45, leg_s) + draw_legend_box (ctx, "Install", \ + TASK_COLOR_INSTALL, off_x+240, curr_y + 45, leg_s) + draw_legend_box (ctx, "Populate Sysroot", \ + TASK_COLOR_SYSROOT, off_x+360, curr_y + 45, leg_s) + draw_legend_box (ctx, "Package", \ + TASK_COLOR_PACKAGE, off_x+480, curr_y + 45, leg_s) + draw_legend_box (ctx, "Package Write", + TASK_COLOR_PACKAGE_WRITE, off_x+600, curr_y + 45, leg_s) + + ctx.set_font_size(PROC_TEXT_FONT_SIZE) + + draw_box_ticks(ctx, chart_rect, sec_w) + draw_sec_labels(ctx, options, chart_rect, sec_w, 30) + + y = curr_y+header_h + + offset = trace.min or min(trace.start.keys()) + for s in sorted(trace.start.keys()): + for val in sorted(trace.start[s]): + if not options.app_options.show_all and \ + trace.processes[val][1] - s < options.app_options.mintime: + continue + task = val.split(":")[1] + #print val + #print trace.processes[val][1] + #print s + x = chart_rect[0] + (s - offset) * sec_w + w = ((trace.processes[val][1] - s) * sec_w) + + #print "proc at %s %s %s %s" % (x, y, w, proc_h) + col = None + if task == "do_compile": + col = TASK_COLOR_COMPILE + elif task == "do_configure": + col = TASK_COLOR_CONFIGURE + elif task == "do_install": + col = TASK_COLOR_INSTALL + elif task == "do_populate_sysroot": + col = TASK_COLOR_SYSROOT + elif task == "do_package": + col = TASK_COLOR_PACKAGE + elif task == "do_package_write_rpm" or \ + task == "do_package_write_deb" or \ + task == "do_package_write_ipk": + col = TASK_COLOR_PACKAGE_WRITE + else: + col = WHITE + + if col: + draw_fill_rect(ctx, col, (x, y, w, proc_h)) + draw_rect(ctx, PROC_BORDER_COLOR, (x, y, w, proc_h)) + + draw_label_in_box(ctx, PROC_TEXT_COLOR, val, x, y + proc_h - 4, w, proc_h) + y = y + proc_h + + return curr_y + +# +# Render the chart. +# +def render(ctx, options, xscale, trace): + (w, h) = extents (options, xscale, trace) + global OPTIONS + OPTIONS = options.app_options + + # x, y, w, h + clip = ctx.clip_extents() + + sec_w = int (xscale * sec_w_base) + ctx.set_line_width(1.0) + ctx.select_font_face(FONT_NAME) + draw_fill_rect(ctx, WHITE, (0, 0, max(w, MIN_IMG_W), h)) + w -= 2*off_x + curr_y = off_y; + + curr_y = render_processes_chart (ctx, options, trace, curr_y, w, h, sec_w) + + return + + proc_tree = options.proc_tree (trace) + + # draw the title and headers + if proc_tree.idle: + duration = proc_tree.idle + else: + duration = proc_tree.duration + + if not options.kernel_only: + curr_y = draw_header (ctx, trace.headers, duration) + else: + curr_y = off_y; + + if options.charts: + curr_y = render_charts (ctx, options, clip, trace, curr_y, w, h, sec_w) + + # draw process boxes + proc_height = h + if proc_tree.taskstats and options.cumulative: + proc_height -= CUML_HEIGHT + + draw_process_bar_chart(ctx, clip, options, proc_tree, trace.times, + curr_y, w, proc_height, sec_w) + + curr_y = proc_height + ctx.set_font_size(SIG_FONT_SIZE) + draw_text(ctx, SIGNATURE, SIG_COLOR, off_x + 5, proc_height - 8) + + # draw a cumulative CPU-time-per-process graph + if proc_tree.taskstats and options.cumulative: + cuml_rect = (off_x, curr_y + off_y, w, CUML_HEIGHT/2 - off_y * 2) + if clip_visible (clip, cuml_rect): + draw_cuml_graph(ctx, proc_tree, cuml_rect, duration, sec_w, STAT_TYPE_CPU) + + # draw a cumulative I/O-time-per-process graph + if proc_tree.taskstats and options.cumulative: + cuml_rect = (off_x, curr_y + off_y * 100, w, CUML_HEIGHT/2 - off_y * 2) + if clip_visible (clip, cuml_rect): + draw_cuml_graph(ctx, proc_tree, cuml_rect, duration, sec_w, STAT_TYPE_IO) + +def draw_process_bar_chart(ctx, clip, options, proc_tree, times, curr_y, w, h, sec_w): + header_size = 0 + if not options.kernel_only: + draw_legend_box (ctx, "Running (%cpu)", + PROC_COLOR_R, off_x , curr_y + 45, leg_s) + draw_legend_box (ctx, "Unint.sleep (I/O)", + PROC_COLOR_D, off_x+120, curr_y + 45, leg_s) + draw_legend_box (ctx, "Sleeping", + PROC_COLOR_S, off_x+240, curr_y + 45, leg_s) + draw_legend_box (ctx, "Zombie", + PROC_COLOR_Z, off_x+360, curr_y + 45, leg_s) + header_size = 45 + + chart_rect = [off_x, curr_y + header_size + 15, + w, h - 2 * off_y - (curr_y + header_size + 15) + proc_h] + ctx.set_font_size (PROC_TEXT_FONT_SIZE) + + draw_box_ticks (ctx, chart_rect, sec_w) + if sec_w > 100: + nsec = 1 + else: + nsec = 5 + draw_sec_labels (ctx, options, chart_rect, sec_w, nsec) + draw_annotations (ctx, proc_tree, times, chart_rect) + + y = curr_y + 60 + for root in proc_tree.process_tree: + draw_processes_recursively(ctx, root, proc_tree, y, proc_h, chart_rect, clip) + y = y + proc_h * proc_tree.num_nodes([root]) + + +def draw_header (ctx, headers, duration): + toshow = [ + ('system.uname', 'uname', lambda s: s), + ('system.release', 'release', lambda s: s), + ('system.cpu', 'CPU', lambda s: re.sub('model name\s*:\s*', '', s, 1)), + ('system.kernel.options', 'kernel options', lambda s: s), + ] + + header_y = ctx.font_extents()[2] + 10 + ctx.set_font_size(TITLE_FONT_SIZE) + draw_text(ctx, headers['title'], TEXT_COLOR, off_x, header_y) + ctx.set_font_size(TEXT_FONT_SIZE) + + for (headerkey, headertitle, mangle) in toshow: + header_y += ctx.font_extents()[2] + if headerkey in headers: + value = headers.get(headerkey) + else: + value = "" + txt = headertitle + ': ' + mangle(value) + draw_text(ctx, txt, TEXT_COLOR, off_x, header_y) + + dur = duration / 100.0 + txt = 'time : %02d:%05.2f' % (math.floor(dur/60), dur - 60 * math.floor(dur/60)) + if headers.get('system.maxpid') is not None: + txt = txt + ' max pid: %s' % (headers.get('system.maxpid')) + + header_y += ctx.font_extents()[2] + draw_text (ctx, txt, TEXT_COLOR, off_x, header_y) + + return header_y + +def draw_processes_recursively(ctx, proc, proc_tree, y, proc_h, rect, clip) : + x = rect[0] + ((proc.start_time - proc_tree.start_time) * rect[2] / proc_tree.duration) + w = ((proc.duration) * rect[2] / proc_tree.duration) + + draw_process_activity_colors(ctx, proc, proc_tree, x, y, w, proc_h, rect, clip) + draw_rect(ctx, PROC_BORDER_COLOR, (x, y, w, proc_h)) + ipid = int(proc.pid) + if not OPTIONS.show_all: + cmdString = proc.cmd + else: + cmdString = '' + if (OPTIONS.show_pid or OPTIONS.show_all) and ipid is not 0: + cmdString = cmdString + " [" + str(ipid // 1000) + "]" + if OPTIONS.show_all: + if proc.args: + cmdString = cmdString + " '" + "' '".join(proc.args) + "'" + else: + cmdString = cmdString + " " + proc.exe + + draw_label_in_box(ctx, PROC_TEXT_COLOR, cmdString, x, y + proc_h - 4, w, rect[0] + rect[2]) + + next_y = y + proc_h + for child in proc.child_list: + if next_y > clip[1] + clip[3]: + break + child_x, child_y = draw_processes_recursively(ctx, child, proc_tree, next_y, proc_h, rect, clip) + draw_process_connecting_lines(ctx, x, y, child_x, child_y, proc_h) + next_y = next_y + proc_h * proc_tree.num_nodes([child]) + + return x, y + + +def draw_process_activity_colors(ctx, proc, proc_tree, x, y, w, proc_h, rect, clip): + + if y > clip[1] + clip[3] or y + proc_h + 2 < clip[1]: + return + + draw_fill_rect(ctx, PROC_COLOR_S, (x, y, w, proc_h)) + + last_tx = -1 + for sample in proc.samples : + tx = rect[0] + round(((sample.time - proc_tree.start_time) * rect[2] / proc_tree.duration)) + + # samples are sorted chronologically + if tx < clip[0]: + continue + if tx > clip[0] + clip[2]: + break + + tw = round(proc_tree.sample_period * rect[2] / float(proc_tree.duration)) + if last_tx != -1 and abs(last_tx - tx) <= tw: + tw -= last_tx - tx + tx = last_tx + tw = max (tw, 1) # nice to see at least something + + last_tx = tx + tw + state = get_proc_state( sample.state ) + + color = STATE_COLORS[state] + if state == STATE_RUNNING: + alpha = min (sample.cpu_sample.user + sample.cpu_sample.sys, 1.0) + color = tuple(list(PROC_COLOR_R[0:3]) + [alpha]) +# print "render time %d [ tx %d tw %d ], sample state %s color %s alpha %g" % (sample.time, tx, tw, state, color, alpha) + elif state == STATE_SLEEPING: + continue + + draw_fill_rect(ctx, color, (tx, y, tw, proc_h)) + +def draw_process_connecting_lines(ctx, px, py, x, y, proc_h): + ctx.set_source_rgba(*DEP_COLOR) + ctx.set_dash([2, 2]) + if abs(px - x) < 3: + dep_off_x = 3 + dep_off_y = proc_h / 4 + ctx.move_to(x, y + proc_h / 2) + ctx.line_to(px - dep_off_x, y + proc_h / 2) + ctx.line_to(px - dep_off_x, py - dep_off_y) + ctx.line_to(px, py - dep_off_y) + else: + ctx.move_to(x, y + proc_h / 2) + ctx.line_to(px, y + proc_h / 2) + ctx.line_to(px, py) + ctx.stroke() + ctx.set_dash([]) + +# elide the bootchart collector - it is quite distorting +def elide_bootchart(proc): + return proc.cmd == 'bootchartd' or proc.cmd == 'bootchart-colle' + +class CumlSample: + def __init__(self, proc): + self.cmd = proc.cmd + self.samples = [] + self.merge_samples (proc) + self.color = None + + def merge_samples(self, proc): + self.samples.extend (proc.samples) + self.samples.sort (key = lambda p: p.time) + + def next(self): + global palette_idx + palette_idx += HSV_STEP + return palette_idx + + def get_color(self): + if self.color is None: + i = self.next() % HSV_MAX_MOD + h = 0.0 + if i is not 0: + h = (1.0 * i) / HSV_MAX_MOD + s = 0.5 + v = 1.0 + c = colorsys.hsv_to_rgb (h, s, v) + self.color = (c[0], c[1], c[2], 1.0) + return self.color + + +def draw_cuml_graph(ctx, proc_tree, chart_bounds, duration, sec_w, stat_type): + global palette_idx + palette_idx = 0 + + time_hash = {} + total_time = 0.0 + m_proc_list = {} + + if stat_type is STAT_TYPE_CPU: + sample_value = 'cpu' + else: + sample_value = 'io' + for proc in proc_tree.process_list: + if elide_bootchart(proc): + continue + + for sample in proc.samples: + total_time += getattr(sample.cpu_sample, sample_value) + if not sample.time in time_hash: + time_hash[sample.time] = 1 + + # merge pids with the same cmd + if not proc.cmd in m_proc_list: + m_proc_list[proc.cmd] = CumlSample (proc) + continue + s = m_proc_list[proc.cmd] + s.merge_samples (proc) + + # all the sample times + times = sorted(time_hash) + if len (times) < 2: + print("degenerate boot chart") + return + + pix_per_ns = chart_bounds[3] / total_time +# print "total time: %g pix-per-ns %g" % (total_time, pix_per_ns) + + # FIXME: we have duplicates in the process list too [!] - why !? + + # Render bottom up, left to right + below = {} + for time in times: + below[time] = chart_bounds[1] + chart_bounds[3] + + # same colors each time we render + random.seed (0) + + ctx.set_line_width(1) + + legends = [] + labels = [] + + # render each pid in order + for cs in m_proc_list.values(): + row = {} + cuml = 0.0 + + # print "pid : %s -> %g samples %d" % (proc.cmd, cuml, len (cs.samples)) + for sample in cs.samples: + cuml += getattr(sample.cpu_sample, sample_value) + row[sample.time] = cuml + + process_total_time = cuml + + # hide really tiny processes + if cuml * pix_per_ns <= 2: + continue + + last_time = times[0] + y = last_below = below[last_time] + last_cuml = cuml = 0.0 + + ctx.set_source_rgba(*cs.get_color()) + for time in times: + render_seg = False + + # did the underlying trend increase ? + if below[time] != last_below: + last_below = below[last_time] + last_cuml = cuml + render_seg = True + + # did we move up a pixel increase ? + if time in row: + nc = round (row[time] * pix_per_ns) + if nc != cuml: + last_cuml = cuml + cuml = nc + render_seg = True + +# if last_cuml > cuml: +# assert fail ... - un-sorted process samples + + # draw the trailing rectangle from the last time to + # before now, at the height of the last segment. + if render_seg: + w = math.ceil ((time - last_time) * chart_bounds[2] / proc_tree.duration) + 1 + x = chart_bounds[0] + round((last_time - proc_tree.start_time) * chart_bounds[2] / proc_tree.duration) + ctx.rectangle (x, below[last_time] - last_cuml, w, last_cuml) + ctx.fill() +# ctx.stroke() + last_time = time + y = below [time] - cuml + + row[time] = y + + # render the last segment + x = chart_bounds[0] + round((last_time - proc_tree.start_time) * chart_bounds[2] / proc_tree.duration) + y = below[last_time] - cuml + ctx.rectangle (x, y, chart_bounds[2] - x, cuml) + ctx.fill() +# ctx.stroke() + + # render legend if it will fit + if cuml > 8: + label = cs.cmd + extnts = ctx.text_extents(label) + label_w = extnts[2] + label_h = extnts[3] +# print "Text extents %g by %g" % (label_w, label_h) + labels.append((label, + chart_bounds[0] + chart_bounds[2] - label_w - off_x * 2, + y + (cuml + label_h) / 2)) + if cs in legends: + print("ARGH - duplicate process in list !") + + legends.append ((cs, process_total_time)) + + below = row + + # render grid-lines over the top + draw_box_ticks(ctx, chart_bounds, sec_w) + + # render labels + for l in labels: + draw_text(ctx, l[0], TEXT_COLOR, l[1], l[2]) + + # Render legends + font_height = 20 + label_width = 300 + LEGENDS_PER_COL = 15 + LEGENDS_TOTAL = 45 + ctx.set_font_size (TITLE_FONT_SIZE) + dur_secs = duration / 100 + cpu_secs = total_time / 1000000000 + + # misleading - with multiple CPUs ... +# idle = ((dur_secs - cpu_secs) / dur_secs) * 100.0 + if stat_type is STAT_TYPE_CPU: + label = "Cumulative CPU usage, by process; total CPU: " \ + " %.5g(s) time: %.3g(s)" % (cpu_secs, dur_secs) + else: + label = "Cumulative I/O usage, by process; total I/O: " \ + " %.5g(s) time: %.3g(s)" % (cpu_secs, dur_secs) + + draw_text(ctx, label, TEXT_COLOR, chart_bounds[0] + off_x, + chart_bounds[1] + font_height) + + i = 0 + legends = sorted(legends, key=itemgetter(1), reverse=True) + ctx.set_font_size(TEXT_FONT_SIZE) + for t in legends: + cs = t[0] + time = t[1] + x = chart_bounds[0] + off_x + int (i/LEGENDS_PER_COL) * label_width + y = chart_bounds[1] + font_height * ((i % LEGENDS_PER_COL) + 2) + str = "%s - %.0f(ms) (%2.2f%%)" % (cs.cmd, time/1000000, (time/total_time) * 100.0) + draw_legend_box(ctx, str, cs.color, x, y, leg_s) + i = i + 1 + if i >= LEGENDS_TOTAL: + break diff --git a/scripts/pybootchartgui/pybootchartgui/gui.py b/scripts/pybootchartgui/pybootchartgui/gui.py new file mode 100644 index 0000000..7fedd23 --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/gui.py @@ -0,0 +1,350 @@ +# This file is part of pybootchartgui. + +# pybootchartgui is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# pybootchartgui is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with pybootchartgui. If not, see <http://www.gnu.org/licenses/>. + +import gobject +import gtk +import gtk.gdk +import gtk.keysyms +from . import draw +from .draw import RenderOptions + +class PyBootchartWidget(gtk.DrawingArea): + __gsignals__ = { + 'expose-event': 'override', + 'clicked' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, (gobject.TYPE_STRING, gtk.gdk.Event)), + 'position-changed' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, (gobject.TYPE_INT, gobject.TYPE_INT)), + 'set-scroll-adjustments' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, (gtk.Adjustment, gtk.Adjustment)) + } + + def __init__(self, trace, options, xscale): + gtk.DrawingArea.__init__(self) + + self.trace = trace + self.options = options + + self.set_flags(gtk.CAN_FOCUS) + + self.add_events(gtk.gdk.BUTTON_PRESS_MASK | gtk.gdk.BUTTON_RELEASE_MASK) + self.connect("button-press-event", self.on_area_button_press) + self.connect("button-release-event", self.on_area_button_release) + self.add_events(gtk.gdk.POINTER_MOTION_MASK | gtk.gdk.POINTER_MOTION_HINT_MASK | gtk.gdk.BUTTON_RELEASE_MASK) + self.connect("motion-notify-event", self.on_area_motion_notify) + self.connect("scroll-event", self.on_area_scroll_event) + self.connect('key-press-event', self.on_key_press_event) + + self.connect('set-scroll-adjustments', self.on_set_scroll_adjustments) + self.connect("size-allocate", self.on_allocation_size_changed) + self.connect("position-changed", self.on_position_changed) + + self.zoom_ratio = 1.0 + self.xscale = xscale + self.x, self.y = 0.0, 0.0 + + self.chart_width, self.chart_height = draw.extents(self.options, self.xscale, self.trace) + self.hadj = None + self.vadj = None + self.hadj_changed_signal_id = None + self.vadj_changed_signal_id = None + + def do_expose_event(self, event): + cr = self.window.cairo_create() + + # set a clip region for the expose event + cr.rectangle( + event.area.x, event.area.y, + event.area.width, event.area.height + ) + cr.clip() + self.draw(cr, self.get_allocation()) + return False + + def draw(self, cr, rect): + cr.set_source_rgba(1.0, 1.0, 1.0, 1.0) + cr.paint() + cr.scale(self.zoom_ratio, self.zoom_ratio) + cr.translate(-self.x, -self.y) + draw.render(cr, self.options, self.xscale, self.trace) + + def position_changed(self): + self.emit("position-changed", self.x, self.y) + + ZOOM_INCREMENT = 1.25 + + def zoom_image (self, zoom_ratio): + self.zoom_ratio = zoom_ratio + self._set_scroll_adjustments (self.hadj, self.vadj) + self.queue_draw() + + def zoom_to_rect (self, rect): + zoom_ratio = float(rect.width)/float(self.chart_width) + self.zoom_image(zoom_ratio) + self.x = 0 + self.position_changed() + + def set_xscale(self, xscale): + old_mid_x = self.x + self.hadj.page_size / 2 + self.xscale = xscale + self.chart_width, self.chart_height = draw.extents(self.options, self.xscale, self.trace) + new_x = old_mid_x + self.zoom_image (self.zoom_ratio) + + def on_expand(self, action): + self.set_xscale (int(self.xscale * 1.5 + 0.5)) + + def on_contract(self, action): + self.set_xscale (max(int(self.xscale / 1.5), 1)) + + def on_zoom_in(self, action): + self.zoom_image(self.zoom_ratio * self.ZOOM_INCREMENT) + + def on_zoom_out(self, action): + self.zoom_image(self.zoom_ratio / self.ZOOM_INCREMENT) + + def on_zoom_fit(self, action): + self.zoom_to_rect(self.get_allocation()) + + def on_zoom_100(self, action): + self.zoom_image(1.0) + self.set_xscale(1.0) + + def show_toggled(self, button): + self.options.app_options.show_all = button.get_property ('active') + self.chart_width, self.chart_height = draw.extents(self.options, self.xscale, self.trace) + self._set_scroll_adjustments(self.hadj, self.vadj) + self.queue_draw() + + POS_INCREMENT = 100 + + def on_key_press_event(self, widget, event): + if event.keyval == gtk.keysyms.Left: + self.x -= self.POS_INCREMENT/self.zoom_ratio + elif event.keyval == gtk.keysyms.Right: + self.x += self.POS_INCREMENT/self.zoom_ratio + elif event.keyval == gtk.keysyms.Up: + self.y -= self.POS_INCREMENT/self.zoom_ratio + elif event.keyval == gtk.keysyms.Down: + self.y += self.POS_INCREMENT/self.zoom_ratio + else: + return False + self.queue_draw() + self.position_changed() + return True + + def on_area_button_press(self, area, event): + if event.button == 2 or event.button == 1: + area.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.FLEUR)) + self.prevmousex = event.x + self.prevmousey = event.y + if event.type not in (gtk.gdk.BUTTON_PRESS, gtk.gdk.BUTTON_RELEASE): + return False + return False + + def on_area_button_release(self, area, event): + if event.button == 2 or event.button == 1: + area.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.ARROW)) + self.prevmousex = None + self.prevmousey = None + return True + return False + + def on_area_scroll_event(self, area, event): + if event.state & gtk.gdk.CONTROL_MASK: + if event.direction == gtk.gdk.SCROLL_UP: + self.zoom_image(self.zoom_ratio * self.ZOOM_INCREMENT) + return True + if event.direction == gtk.gdk.SCROLL_DOWN: + self.zoom_image(self.zoom_ratio / self.ZOOM_INCREMENT) + return True + return False + + def on_area_motion_notify(self, area, event): + state = event.state + if state & gtk.gdk.BUTTON2_MASK or state & gtk.gdk.BUTTON1_MASK: + x, y = int(event.x), int(event.y) + # pan the image + self.x += (self.prevmousex - x)/self.zoom_ratio + self.y += (self.prevmousey - y)/self.zoom_ratio + self.queue_draw() + self.prevmousex = x + self.prevmousey = y + self.position_changed() + return True + + def on_set_scroll_adjustments(self, area, hadj, vadj): + self._set_scroll_adjustments (hadj, vadj) + + def on_allocation_size_changed(self, widget, allocation): + self.hadj.page_size = allocation.width + self.hadj.page_increment = allocation.width * 0.9 + self.vadj.page_size = allocation.height + self.vadj.page_increment = allocation.height * 0.9 + + def _set_adj_upper(self, adj, upper): + changed = False + value_changed = False + + if adj.upper != upper: + adj.upper = upper + changed = True + + max_value = max(0.0, upper - adj.page_size) + if adj.value > max_value: + adj.value = max_value + value_changed = True + + if changed: + adj.changed() + if value_changed: + adj.value_changed() + + def _set_scroll_adjustments(self, hadj, vadj): + if hadj == None: + hadj = gtk.Adjustment(0.0, 0.0, 0.0, 0.0, 0.0, 0.0) + if vadj == None: + vadj = gtk.Adjustment(0.0, 0.0, 0.0, 0.0, 0.0, 0.0) + + if self.hadj_changed_signal_id != None and \ + self.hadj != None and hadj != self.hadj: + self.hadj.disconnect (self.hadj_changed_signal_id) + if self.vadj_changed_signal_id != None and \ + self.vadj != None and vadj != self.vadj: + self.vadj.disconnect (self.vadj_changed_signal_id) + + if hadj != None: + self.hadj = hadj + self._set_adj_upper (self.hadj, self.zoom_ratio * self.chart_width) + self.hadj_changed_signal_id = self.hadj.connect('value-changed', self.on_adjustments_changed) + + if vadj != None: + self.vadj = vadj + self._set_adj_upper (self.vadj, self.zoom_ratio * self.chart_height) + self.vadj_changed_signal_id = self.vadj.connect('value-changed', self.on_adjustments_changed) + + def on_adjustments_changed(self, adj): + self.x = self.hadj.value / self.zoom_ratio + self.y = self.vadj.value / self.zoom_ratio + self.queue_draw() + + def on_position_changed(self, widget, x, y): + self.hadj.value = x * self.zoom_ratio + self.vadj.value = y * self.zoom_ratio + +PyBootchartWidget.set_set_scroll_adjustments_signal('set-scroll-adjustments') + +class PyBootchartShell(gtk.VBox): + ui = ''' + <ui> + <toolbar name="ToolBar"> + <toolitem action="Expand"/> + <toolitem action="Contract"/> + <separator/> + <toolitem action="ZoomIn"/> + <toolitem action="ZoomOut"/> + <toolitem action="ZoomFit"/> + <toolitem action="Zoom100"/> + </toolbar> + </ui> + ''' + def __init__(self, window, trace, options, xscale): + gtk.VBox.__init__(self) + + self.widget = PyBootchartWidget(trace, options, xscale) + + # Create a UIManager instance + uimanager = self.uimanager = gtk.UIManager() + + # Add the accelerator group to the toplevel window + accelgroup = uimanager.get_accel_group() + window.add_accel_group(accelgroup) + + # Create an ActionGroup + actiongroup = gtk.ActionGroup('Actions') + self.actiongroup = actiongroup + + # Create actions + actiongroup.add_actions(( + ('Expand', gtk.STOCK_ADD, None, None, None, self.widget.on_expand), + ('Contract', gtk.STOCK_REMOVE, None, None, None, self.widget.on_contract), + ('ZoomIn', gtk.STOCK_ZOOM_IN, None, None, None, self.widget.on_zoom_in), + ('ZoomOut', gtk.STOCK_ZOOM_OUT, None, None, None, self.widget.on_zoom_out), + ('ZoomFit', gtk.STOCK_ZOOM_FIT, 'Fit Width', None, None, self.widget.on_zoom_fit), + ('Zoom100', gtk.STOCK_ZOOM_100, None, None, None, self.widget.on_zoom_100), + )) + + # Add the actiongroup to the uimanager + uimanager.insert_action_group(actiongroup, 0) + + # Add a UI description + uimanager.add_ui_from_string(self.ui) + + # Scrolled window + scrolled = gtk.ScrolledWindow() + scrolled.add(self.widget) + + # toolbar / h-box + hbox = gtk.HBox(False, 8) + + # Create a Toolbar + toolbar = uimanager.get_widget('/ToolBar') + hbox.pack_start(toolbar, True, True) + + if not options.kernel_only: + # Misc. options + button = gtk.CheckButton("Show more") + button.connect ('toggled', self.widget.show_toggled) + button.set_active(options.app_options.show_all) + hbox.pack_start (button, False, True) + + self.pack_start(hbox, False) + self.pack_start(scrolled) + self.show_all() + + def grab_focus(self, window): + window.set_focus(self.widget) + + +class PyBootchartWindow(gtk.Window): + + def __init__(self, trace, app_options): + gtk.Window.__init__(self) + + window = self + window.set_title("Bootchart %s" % trace.filename) + window.set_default_size(750, 550) + + tab_page = gtk.Notebook() + tab_page.show() + window.add(tab_page) + + full_opts = RenderOptions(app_options) + full_tree = PyBootchartShell(window, trace, full_opts, 1.0) + tab_page.append_page (full_tree, gtk.Label("Full tree")) + + if trace.kernel is not None and len (trace.kernel) > 2: + kernel_opts = RenderOptions(app_options) + kernel_opts.cumulative = False + kernel_opts.charts = False + kernel_opts.kernel_only = True + kernel_tree = PyBootchartShell(window, trace, kernel_opts, 5.0) + tab_page.append_page (kernel_tree, gtk.Label("Kernel boot")) + + full_tree.grab_focus(self) + self.show() + + +def show(trace, options): + win = PyBootchartWindow(trace, options) + win.connect('destroy', gtk.main_quit) + gtk.main() diff --git a/scripts/pybootchartgui/pybootchartgui/main.py b/scripts/pybootchartgui/pybootchartgui/main.py new file mode 120000 index 0000000..b45ae0a --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/main.py @@ -0,0 +1 @@ +main.py.in
\ No newline at end of file diff --git a/scripts/pybootchartgui/pybootchartgui/main.py.in b/scripts/pybootchartgui/pybootchartgui/main.py.in new file mode 100644 index 0000000..21bb0be --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/main.py.in @@ -0,0 +1,187 @@ +# +# *********************************************************************** +# Warning: This file is auto-generated from main.py.in - edit it there. +# *********************************************************************** +# +# pybootchartgui is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# pybootchartgui is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with pybootchartgui. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import print_function + +import sys +import os +import optparse + +from . import parsing +from . import batch + +def _mk_options_parser(): + """Make an options parser.""" + usage = "%prog [options] /path/to/tmp/buildstats/<recipe-machine>/<BUILDNAME>/" + version = "%prog v1.0.0" + parser = optparse.OptionParser(usage, version=version) + parser.add_option("-i", "--interactive", action="store_true", dest="interactive", default=False, + help="start in active mode") + parser.add_option("-f", "--format", dest="format", default="png", choices=["png", "svg", "pdf"], + help="image format (png, svg, pdf); default format png") + parser.add_option("-o", "--output", dest="output", metavar="PATH", default=None, + help="output path (file or directory) where charts are stored") + parser.add_option("-s", "--split", dest="num", type=int, default=1, + help="split the output chart into <NUM> charts, only works with \"-o PATH\"") + parser.add_option("-m", "--mintime", dest="mintime", type=int, default=8, + help="only tasks longer than this time will be displayed") + parser.add_option("-M", "--minutes", action="store_true", dest="as_minutes", default=False, + help="display time in minutes instead of seconds") +# parser.add_option("-n", "--no-prune", action="store_false", dest="prune", default=True, +# help="do not prune the process tree") + parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, + help="suppress informational messages") +# parser.add_option("-t", "--boot-time", action="store_true", dest="boottime", default=False, +# help="only display the boot time of the boot in text format (stdout)") + parser.add_option("--very-quiet", action="store_true", dest="veryquiet", default=False, + help="suppress all messages except errors") + parser.add_option("--verbose", action="store_true", dest="verbose", default=False, + help="print all messages") +# parser.add_option("--profile", action="store_true", dest="profile", default=False, +# help="profile rendering of chart (only useful when in batch mode indicated by -f)") +# parser.add_option("--show-pid", action="store_true", dest="show_pid", default=False, +# help="show process ids in the bootchart as 'processname [pid]'") + parser.add_option("--show-all", action="store_true", dest="show_all", default=False, + help="show all processes in the chart") +# parser.add_option("--crop-after", dest="crop_after", metavar="PROCESS", default=None, +# help="crop chart when idle after PROCESS is started") +# parser.add_option("--annotate", action="append", dest="annotate", metavar="PROCESS", default=None, +# help="annotate position where PROCESS is started; can be specified multiple times. " + +# "To create a single annotation when any one of a set of processes is started, use commas to separate the names") +# parser.add_option("--annotate-file", dest="annotate_file", metavar="FILENAME", default=None, +# help="filename to write annotation points to") + parser.add_option("-T", "--full-time", action="store_true", dest="full_time", default=False, + help="display the full time regardless of which processes are currently shown") + return parser + +class Writer: + def __init__(self, write, options): + self.write = write + self.options = options + + def error(self, msg): + self.write(msg) + + def warn(self, msg): + if not self.options.quiet: + self.write(msg) + + def info(self, msg): + if self.options.verbose: + self.write(msg) + + def status(self, msg): + if not self.options.quiet: + self.write(msg) + +def _mk_writer(options): + def write(s): + print(s) + return Writer(write, options) + +def _get_filename(path): + """Construct a usable filename for outputs""" + dname = "." + fname = "bootchart" + if path != None: + if os.path.isdir(path): + dname = path + else: + fname = path + return os.path.join(dname, fname) + +def main(argv=None): + try: + if argv is None: + argv = sys.argv[1:] + + parser = _mk_options_parser() + options, args = parser.parse_args(argv) + + # Default values for disabled options + options.prune = True + options.boottime = False + options.profile = False + options.show_pid = False + options.crop_after = None + options.annotate = None + options.annotate_file = None + + writer = _mk_writer(options) + + if len(args) == 0: + print("No path given, trying /var/log/bootchart.tgz") + args = [ "/var/log/bootchart.tgz" ] + + res = parsing.Trace(writer, args, options) + + if options.interactive or options.output == None: + from . import gui + gui.show(res, options) + elif options.boottime: + import math + proc_tree = res.proc_tree + if proc_tree.idle: + duration = proc_tree.idle + else: + duration = proc_tree.duration + dur = duration / 100.0 + print('%02d:%05.2f' % (math.floor(dur/60), dur - 60 * math.floor(dur/60))) + else: + if options.annotate_file: + f = open (options.annotate_file, "w") + try: + for time in res[4]: + if time is not None: + # output as ms + print(time * 10, file=f) + else: + print(file=f) + finally: + f.close() + filename = _get_filename(options.output) + res_list = parsing.split_res(res, options) + n = 1 + width = len(str(len(res_list))) + s = "_%%0%dd." % width + for r in res_list: + if len(res_list) == 1: + f = filename + "." + options.format + else: + f = filename + s % n + options.format + n = n + 1 + def render(): + batch.render(writer, r, options, f) + if options.profile: + import cProfile + import pstats + profile = '%s.prof' % os.path.splitext(filename)[0] + cProfile.runctx('render()', globals(), locals(), profile) + p = pstats.Stats(profile) + p.strip_dirs().sort_stats('time').print_stats(20) + else: + render() + + return 0 + except parsing.ParseError as ex: + print(("Parse error: %s" % ex)) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/pybootchartgui/pybootchartgui/parsing.py b/scripts/pybootchartgui/pybootchartgui/parsing.py new file mode 100644 index 0000000..d423b9f --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/parsing.py @@ -0,0 +1,740 @@ +# This file is part of pybootchartgui. + +# pybootchartgui is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# pybootchartgui is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with pybootchartgui. If not, see <http://www.gnu.org/licenses/>. + + +from __future__ import with_statement + +import os +import string +import re +import sys +import tarfile +from time import clock +from collections import defaultdict +from functools import reduce + +from .samples import * +from .process_tree import ProcessTree + +if sys.version_info >= (3, 0): + long = int + +# Parsing produces as its end result a 'Trace' + +class Trace: + def __init__(self, writer, paths, options): + self.processes = {} + self.start = {} + self.end = {} + self.min = None + self.max = None + self.headers = None + self.disk_stats = None + self.ps_stats = None + self.taskstats = None + self.cpu_stats = None + self.cmdline = None + self.kernel = None + self.kernel_tree = None + self.filename = None + self.parent_map = None + self.mem_stats = None + + if len(paths): + parse_paths (writer, self, paths) + if not self.valid(): + raise ParseError("empty state: '%s' does not contain a valid bootchart" % ", ".join(paths)) + + if options.full_time: + self.min = min(self.start.keys()) + self.max = max(self.end.keys()) + + return + + # Turn that parsed information into something more useful + # link processes into a tree of pointers, calculate statistics + self.compile(writer) + + # Crop the chart to the end of the first idle period after the given + # process + if options.crop_after: + idle = self.crop (writer, options.crop_after) + else: + idle = None + + # Annotate other times as the first start point of given process lists + self.times = [ idle ] + if options.annotate: + for procnames in options.annotate: + names = [x[:15] for x in procnames.split(",")] + for proc in self.ps_stats.process_map.values(): + if proc.cmd in names: + self.times.append(proc.start_time) + break + else: + self.times.append(None) + + self.proc_tree = ProcessTree(writer, self.kernel, self.ps_stats, + self.ps_stats.sample_period, + self.headers.get("profile.process"), + options.prune, idle, self.taskstats, + self.parent_map is not None) + + if self.kernel is not None: + self.kernel_tree = ProcessTree(writer, self.kernel, None, 0, + self.headers.get("profile.process"), + False, None, None, True) + + def valid(self): + return len(self.processes) != 0 + return self.headers != None and self.disk_stats != None and \ + self.ps_stats != None and self.cpu_stats != None + + def add_process(self, process, start, end): + self.processes[process] = [start, end] + if start not in self.start: + self.start[start] = [] + if process not in self.start[start]: + self.start[start].append(process) + if end not in self.end: + self.end[end] = [] + if process not in self.end[end]: + self.end[end].append(process) + + def compile(self, writer): + + def find_parent_id_for(pid): + if pid is 0: + return 0 + ppid = self.parent_map.get(pid) + if ppid: + # many of these double forks are so short lived + # that we have no samples, or process info for them + # so climb the parent hierarcy to find one + if int (ppid * 1000) not in self.ps_stats.process_map: +# print "Pid '%d' short lived with no process" % ppid + ppid = find_parent_id_for (ppid) +# else: +# print "Pid '%d' has an entry" % ppid + else: +# print "Pid '%d' missing from pid map" % pid + return 0 + return ppid + + # merge in the cmdline data + if self.cmdline is not None: + for proc in self.ps_stats.process_map.values(): + rpid = int (proc.pid // 1000) + if rpid in self.cmdline: + cmd = self.cmdline[rpid] + proc.exe = cmd['exe'] + proc.args = cmd['args'] +# else: +# print "proc %d '%s' not in cmdline" % (rpid, proc.exe) + + # re-parent any stray orphans if we can + if self.parent_map is not None: + for process in self.ps_stats.process_map.values(): + ppid = find_parent_id_for (int(process.pid // 1000)) + if ppid: + process.ppid = ppid * 1000 + + # stitch the tree together with pointers + for process in self.ps_stats.process_map.values(): + process.set_parent (self.ps_stats.process_map) + + # count on fingers variously + for process in self.ps_stats.process_map.values(): + process.calc_stats (self.ps_stats.sample_period) + + def crop(self, writer, crop_after): + + def is_idle_at(util, start, j): + k = j + 1 + while k < len(util) and util[k][0] < start + 300: + k += 1 + k = min(k, len(util)-1) + + if util[j][1] >= 0.25: + return False + + avgload = sum(u[1] for u in util[j:k+1]) / (k-j+1) + if avgload < 0.25: + return True + else: + return False + def is_idle(util, start): + for j in range(0, len(util)): + if util[j][0] < start: + continue + return is_idle_at(util, start, j) + else: + return False + + names = [x[:15] for x in crop_after.split(",")] + for proc in self.ps_stats.process_map.values(): + if proc.cmd in names or proc.exe in names: + writer.info("selected proc '%s' from list (start %d)" + % (proc.cmd, proc.start_time)) + break + if proc is None: + writer.warn("no selected crop proc '%s' in list" % crop_after) + + + cpu_util = [(sample.time, sample.user + sample.sys + sample.io) for sample in self.cpu_stats] + disk_util = [(sample.time, sample.util) for sample in self.disk_stats] + + idle = None + for i in range(0, len(cpu_util)): + if cpu_util[i][0] < proc.start_time: + continue + if is_idle_at(cpu_util, cpu_util[i][0], i) \ + and is_idle(disk_util, cpu_util[i][0]): + idle = cpu_util[i][0] + break + + if idle is None: + writer.warn ("not idle after proc '%s'" % crop_after) + return None + + crop_at = idle + 300 + writer.info ("cropping at time %d" % crop_at) + while len (self.cpu_stats) \ + and self.cpu_stats[-1].time > crop_at: + self.cpu_stats.pop() + while len (self.disk_stats) \ + and self.disk_stats[-1].time > crop_at: + self.disk_stats.pop() + + self.ps_stats.end_time = crop_at + + cropped_map = {} + for key, value in self.ps_stats.process_map.items(): + if (value.start_time <= crop_at): + cropped_map[key] = value + + for proc in cropped_map.values(): + proc.duration = min (proc.duration, crop_at - proc.start_time) + while len (proc.samples) \ + and proc.samples[-1].time > crop_at: + proc.samples.pop() + + self.ps_stats.process_map = cropped_map + + return idle + + + +class ParseError(Exception): + """Represents errors during parse of the bootchart.""" + def __init__(self, value): + self.value = value + + def __str__(self): + return self.value + +def _parse_headers(file): + """Parses the headers of the bootchart.""" + def parse(acc, line): + (headers, last) = acc + if '=' in line: + last, value = map (lambda x: x.strip(), line.split('=', 1)) + else: + value = line.strip() + headers[last] += value + return headers, last + return reduce(parse, file.read().decode('utf-8').split('\n'), (defaultdict(str),''))[0] + +def _parse_timed_blocks(file): + """Parses (ie., splits) a file into so-called timed-blocks. A + timed-block consists of a timestamp on a line by itself followed + by zero or more lines of data for that point in time.""" + def parse(block): + lines = block.split('\n') + if not lines: + raise ParseError('expected a timed-block consisting a timestamp followed by data lines') + try: + return (int(lines[0]), lines[1:]) + except ValueError: + raise ParseError("expected a timed-block, but timestamp '%s' is not an integer" % lines[0]) + blocks = file.read().decode('utf-8').split('\n\n') + return [parse(block) for block in blocks if block.strip() and not block.endswith(' not running\n')] + +def _parse_proc_ps_log(writer, file): + """ + * See proc(5) for details. + * + * {pid, comm, state, ppid, pgrp, session, tty_nr, tpgid, flags, minflt, cminflt, majflt, cmajflt, utime, stime, + * cutime, cstime, priority, nice, 0, itrealvalue, starttime, vsize, rss, rlim, startcode, endcode, startstack, + * kstkesp, kstkeip} + """ + processMap = {} + ltime = 0 + timed_blocks = _parse_timed_blocks(file) + for time, lines in timed_blocks: + for line in lines: + if not line: continue + tokens = line.split(' ') + if len(tokens) < 21: + continue + + offset = [index for index, token in enumerate(tokens[1:]) if token[-1] == ')'][0] + pid, cmd, state, ppid = int(tokens[0]), ' '.join(tokens[1:2+offset]), tokens[2+offset], int(tokens[3+offset]) + userCpu, sysCpu, stime = int(tokens[13+offset]), int(tokens[14+offset]), int(tokens[21+offset]) + + # magic fixed point-ness ... + pid *= 1000 + ppid *= 1000 + if pid in processMap: + process = processMap[pid] + process.cmd = cmd.strip('()') # why rename after latest name?? + else: + process = Process(writer, pid, cmd.strip('()'), ppid, min(time, stime)) + processMap[pid] = process + + if process.last_user_cpu_time is not None and process.last_sys_cpu_time is not None and ltime is not None: + userCpuLoad, sysCpuLoad = process.calc_load(userCpu, sysCpu, max(1, time - ltime)) + cpuSample = CPUSample('null', userCpuLoad, sysCpuLoad, 0.0) + process.samples.append(ProcessSample(time, state, cpuSample)) + + process.last_user_cpu_time = userCpu + process.last_sys_cpu_time = sysCpu + ltime = time + + if len (timed_blocks) < 2: + return None + + startTime = timed_blocks[0][0] + avgSampleLength = (ltime - startTime)/(len (timed_blocks) - 1) + + return ProcessStats (writer, processMap, len (timed_blocks), avgSampleLength, startTime, ltime) + +def _parse_taskstats_log(writer, file): + """ + * See bootchart-collector.c for details. + * + * { pid, ppid, comm, cpu_run_real_total, blkio_delay_total, swapin_delay_total } + * + """ + processMap = {} + pidRewrites = {} + ltime = None + timed_blocks = _parse_timed_blocks(file) + for time, lines in timed_blocks: + # we have no 'stime' from taskstats, so prep 'init' + if ltime is None: + process = Process(writer, 1, '[init]', 0, 0) + processMap[1000] = process + ltime = time +# continue + for line in lines: + if not line: continue + tokens = line.split(' ') + if len(tokens) != 6: + continue + + opid, ppid, cmd = int(tokens[0]), int(tokens[1]), tokens[2] + cpu_ns, blkio_delay_ns, swapin_delay_ns = long(tokens[-3]), long(tokens[-2]), long(tokens[-1]), + + # make space for trees of pids + opid *= 1000 + ppid *= 1000 + + # when the process name changes, we re-write the pid. + if opid in pidRewrites: + pid = pidRewrites[opid] + else: + pid = opid + + cmd = cmd.strip('(').strip(')') + if pid in processMap: + process = processMap[pid] + if process.cmd != cmd: + pid += 1 + pidRewrites[opid] = pid +# print "process mutation ! '%s' vs '%s' pid %s -> pid %s\n" % (process.cmd, cmd, opid, pid) + process = process.split (writer, pid, cmd, ppid, time) + processMap[pid] = process + else: + process.cmd = cmd; + else: + process = Process(writer, pid, cmd, ppid, time) + processMap[pid] = process + + delta_cpu_ns = (float) (cpu_ns - process.last_cpu_ns) + delta_blkio_delay_ns = (float) (blkio_delay_ns - process.last_blkio_delay_ns) + delta_swapin_delay_ns = (float) (swapin_delay_ns - process.last_swapin_delay_ns) + + # make up some state data ... + if delta_cpu_ns > 0: + state = "R" + elif delta_blkio_delay_ns + delta_swapin_delay_ns > 0: + state = "D" + else: + state = "S" + + # retain the ns timing information into a CPUSample - that tries + # with the old-style to be a %age of CPU used in this time-slice. + if delta_cpu_ns + delta_blkio_delay_ns + delta_swapin_delay_ns > 0: +# print "proc %s cpu_ns %g delta_cpu %g" % (cmd, cpu_ns, delta_cpu_ns) + cpuSample = CPUSample('null', delta_cpu_ns, 0.0, + delta_blkio_delay_ns, + delta_swapin_delay_ns) + process.samples.append(ProcessSample(time, state, cpuSample)) + + process.last_cpu_ns = cpu_ns + process.last_blkio_delay_ns = blkio_delay_ns + process.last_swapin_delay_ns = swapin_delay_ns + ltime = time + + if len (timed_blocks) < 2: + return None + + startTime = timed_blocks[0][0] + avgSampleLength = (ltime - startTime)/(len(timed_blocks)-1) + + return ProcessStats (writer, processMap, len (timed_blocks), avgSampleLength, startTime, ltime) + +def _parse_proc_stat_log(file): + samples = [] + ltimes = None + for time, lines in _parse_timed_blocks(file): + # skip emtpy lines + if not lines: + continue + # CPU times {user, nice, system, idle, io_wait, irq, softirq} + tokens = lines[0].split() + times = [ int(token) for token in tokens[1:] ] + if ltimes: + user = float((times[0] + times[1]) - (ltimes[0] + ltimes[1])) + system = float((times[2] + times[5] + times[6]) - (ltimes[2] + ltimes[5] + ltimes[6])) + idle = float(times[3] - ltimes[3]) + iowait = float(times[4] - ltimes[4]) + + aSum = max(user + system + idle + iowait, 1) + samples.append( CPUSample(time, user/aSum, system/aSum, iowait/aSum) ) + + ltimes = times + # skip the rest of statistics lines + return samples + +def _parse_proc_disk_stat_log(file, numCpu): + """ + Parse file for disk stats, but only look at the whole device, eg. sda, + not sda1, sda2 etc. The format of relevant lines should be: + {major minor name rio rmerge rsect ruse wio wmerge wsect wuse running use aveq} + """ + disk_regex_re = re.compile ('^([hsv]d.|mtdblock\d|mmcblk\d|cciss/c\d+d\d+.*)$') + + # this gets called an awful lot. + def is_relevant_line(linetokens): + if len(linetokens) != 14: + return False + disk = linetokens[2] + return disk_regex_re.match(disk) + + disk_stat_samples = [] + + for time, lines in _parse_timed_blocks(file): + sample = DiskStatSample(time) + relevant_tokens = [linetokens for linetokens in map (lambda x: x.split(),lines) if is_relevant_line(linetokens)] + + for tokens in relevant_tokens: + disk, rsect, wsect, use = tokens[2], int(tokens[5]), int(tokens[9]), int(tokens[12]) + sample.add_diskdata([rsect, wsect, use]) + + disk_stat_samples.append(sample) + + disk_stats = [] + for sample1, sample2 in zip(disk_stat_samples[:-1], disk_stat_samples[1:]): + interval = sample1.time - sample2.time + if interval == 0: + interval = 1 + sums = [ a - b for a, b in zip(sample1.diskdata, sample2.diskdata) ] + readTput = sums[0] / 2.0 * 100.0 / interval + writeTput = sums[1] / 2.0 * 100.0 / interval + util = float( sums[2] ) / 10 / interval / numCpu + util = max(0.0, min(1.0, util)) + disk_stats.append(DiskSample(sample2.time, readTput, writeTput, util)) + + return disk_stats + +def _parse_proc_meminfo_log(file): + """ + Parse file for global memory statistics. + The format of relevant lines should be: ^key: value( unit)? + """ + used_values = ('MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree',) + + mem_stats = [] + meminfo_re = re.compile(r'([^ \t:]+):\s*(\d+).*') + + for time, lines in _parse_timed_blocks(file): + sample = MemSample(time) + + for line in lines: + match = meminfo_re.match(line) + if not match: + raise ParseError("Invalid meminfo line \"%s\"" % match.groups(0)) + sample.add_value(match.group(1), int(match.group(2))) + + if sample.valid(): + mem_stats.append(sample) + + return mem_stats + +# if we boot the kernel with: initcall_debug printk.time=1 we can +# get all manner of interesting data from the dmesg output +# We turn this into a pseudo-process tree: each event is +# characterised by a +# we don't try to detect a "kernel finished" state - since the kernel +# continues to do interesting things after init is called. +# +# sample input: +# [ 0.000000] ACPI: FACP 3f4fc000 000F4 (v04 INTEL Napa 00000001 MSFT 01000013) +# ... +# [ 0.039993] calling migration_init+0x0/0x6b @ 1 +# [ 0.039993] initcall migration_init+0x0/0x6b returned 1 after 0 usecs +def _parse_dmesg(writer, file): + timestamp_re = re.compile ("^\[\s*(\d+\.\d+)\s*]\s+(.*)$") + split_re = re.compile ("^(\S+)\s+([\S\+_-]+) (.*)$") + processMap = {} + idx = 0 + inc = 1.0 / 1000000 + kernel = Process(writer, idx, "k-boot", 0, 0.1) + processMap['k-boot'] = kernel + base_ts = False + max_ts = 0 + for line in file.read().decode('utf-8').split('\n'): + t = timestamp_re.match (line) + if t is None: +# print "duff timestamp " + line + continue + + time_ms = float (t.group(1)) * 1000 + # looks like we may have a huge diff after the clock + # has been set up. This could lead to huge graph: + # so huge we will be killed by the OOM. + # So instead of using the plain timestamp we will + # use a delta to first one and skip the first one + # for convenience + if max_ts == 0 and not base_ts and time_ms > 1000: + base_ts = time_ms + continue + max_ts = max(time_ms, max_ts) + if base_ts: +# print "fscked clock: used %f instead of %f" % (time_ms - base_ts, time_ms) + time_ms -= base_ts + m = split_re.match (t.group(2)) + + if m is None: + continue +# print "match: '%s'" % (m.group(1)) + type = m.group(1) + func = m.group(2) + rest = m.group(3) + + if t.group(2).startswith ('Write protecting the') or \ + t.group(2).startswith ('Freeing unused kernel memory'): + kernel.duration = time_ms / 10 + continue + +# print "foo: '%s' '%s' '%s'" % (type, func, rest) + if type == "calling": + ppid = kernel.pid + p = re.match ("\@ (\d+)", rest) + if p is not None: + ppid = float (p.group(1)) // 1000 +# print "match: '%s' ('%g') at '%s'" % (func, ppid, time_ms) + name = func.split ('+', 1) [0] + idx += inc + processMap[func] = Process(writer, ppid + idx, name, ppid, time_ms / 10) + elif type == "initcall": +# print "finished: '%s' at '%s'" % (func, time_ms) + if func in processMap: + process = processMap[func] + process.duration = (time_ms / 10) - process.start_time + else: + print("corrupted init call for %s" % (func)) + + elif type == "async_waiting" or type == "async_continuing": + continue # ignore + + return processMap.values() + +# +# Parse binary pacct accounting file output if we have one +# cf. /usr/include/linux/acct.h +# +def _parse_pacct(writer, file): + # read LE int32 + def _read_le_int32(file): + byts = file.read(4) + return (ord(byts[0])) | (ord(byts[1]) << 8) | \ + (ord(byts[2]) << 16) | (ord(byts[3]) << 24) + + parent_map = {} + parent_map[0] = 0 + while file.read(1) != "": # ignore flags + ver = file.read(1) + if ord(ver) < 3: + print("Invalid version 0x%x" % (ord(ver))) + return None + + file.seek (14, 1) # user, group etc. + pid = _read_le_int32 (file) + ppid = _read_le_int32 (file) +# print "Parent of %d is %d" % (pid, ppid) + parent_map[pid] = ppid + file.seek (4 + 4 + 16, 1) # timings + file.seek (16, 1) # acct_comm + return parent_map + +def _parse_paternity_log(writer, file): + parent_map = {} + parent_map[0] = 0 + for line in file.read().decode('utf-8').split('\n'): + if not line: + continue + elems = line.split(' ') # <Child> <Parent> + if len (elems) >= 2: +# print "paternity of %d is %d" % (int(elems[0]), int(elems[1])) + parent_map[int(elems[0])] = int(elems[1]) + else: + print("Odd paternity line '%s'" % (line)) + return parent_map + +def _parse_cmdline_log(writer, file): + cmdLines = {} + for block in file.read().decode('utf-8').split('\n\n'): + lines = block.split('\n') + if len (lines) >= 3: +# print "Lines '%s'" % (lines[0]) + pid = int (lines[0]) + values = {} + values['exe'] = lines[1].lstrip(':') + args = lines[2].lstrip(':').split('\0') + args.pop() + values['args'] = args + cmdLines[pid] = values + return cmdLines + +def get_num_cpus(headers): + """Get the number of CPUs from the system.cpu header property. As the + CPU utilization graphs are relative, the number of CPUs currently makes + no difference.""" + if headers is None: + return 1 + if headers.get("system.cpu.num"): + return max (int (headers.get("system.cpu.num")), 1) + cpu_model = headers.get("system.cpu") + if cpu_model is None: + return 1 + mat = re.match(".*\\((\\d+)\\)", cpu_model) + if mat is None: + return 1 + return max (int(mat.group(1)), 1) + +def _do_parse(writer, state, filename, file): + writer.info("parsing '%s'" % filename) + t1 = clock() + paths = filename.split("/") + task = paths[-1] + pn = paths[-2] + start = None + end = None + for line in file: + if line.startswith("Started:"): + start = int(float(line.split()[-1])) + elif line.startswith("Ended:"): + end = int(float(line.split()[-1])) + if start and end: + state.add_process(pn + ":" + task, start, end) + t2 = clock() + writer.info(" %s seconds" % str(t2-t1)) + return state + +def parse_file(writer, state, filename): + if state.filename is None: + state.filename = filename + basename = os.path.basename(filename) + with open(filename, "rb") as file: + return _do_parse(writer, state, filename, file) + +def parse_paths(writer, state, paths): + for path in paths: + if state.filename is None: + state.filename = path + root, extension = os.path.splitext(path) + if not(os.path.exists(path)): + writer.warn("warning: path '%s' does not exist, ignoring." % path) + continue + #state.filename = path + if os.path.isdir(path): + files = sorted([os.path.join(path, f) for f in os.listdir(path)]) + state = parse_paths(writer, state, files) + elif extension in [".tar", ".tgz", ".gz"]: + if extension == ".gz": + root, extension = os.path.splitext(root) + if extension != ".tar": + writer.warn("warning: can only handle zipped tar files, not zipped '%s'-files; ignoring" % extension) + continue + tf = None + try: + writer.status("parsing '%s'" % path) + tf = tarfile.open(path, 'r:*') + for name in tf.getnames(): + state = _do_parse(writer, state, name, tf.extractfile(name)) + except tarfile.ReadError as error: + raise ParseError("error: could not read tarfile '%s': %s." % (path, error)) + finally: + if tf != None: + tf.close() + else: + state = parse_file(writer, state, path) + return state + +def split_res(res, options): + """ Split the res into n pieces """ + res_list = [] + if options.num > 1: + s_list = sorted(res.start.keys()) + frag_size = len(s_list) / float(options.num) + # Need the top value + if frag_size > int(frag_size): + frag_size = int(frag_size + 1) + else: + frag_size = int(frag_size) + + start = 0 + end = frag_size + while start < end: + state = Trace(None, [], None) + if options.full_time: + state.min = min(res.start.keys()) + state.max = max(res.end.keys()) + for i in range(start, end): + # Add this line for reference + #state.add_process(pn + ":" + task, start, end) + for p in res.start[s_list[i]]: + state.add_process(p, s_list[i], res.processes[p][1]) + start = end + end = end + frag_size + if end > len(s_list): + end = len(s_list) + res_list.append(state) + else: + res_list.append(res) + return res_list diff --git a/scripts/pybootchartgui/pybootchartgui/process_tree.py b/scripts/pybootchartgui/pybootchartgui/process_tree.py new file mode 100644 index 0000000..cf88110 --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/process_tree.py @@ -0,0 +1,292 @@ +# This file is part of pybootchartgui. + +# pybootchartgui is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# pybootchartgui is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with pybootchartgui. If not, see <http://www.gnu.org/licenses/>. + +class ProcessTree: + """ProcessTree encapsulates a process tree. The tree is built from log files + retrieved during the boot process. When building the process tree, it is + pruned and merged in order to be able to visualize it in a comprehensible + manner. + + The following pruning techniques are used: + + * idle processes that keep running during the last process sample + (which is a heuristic for a background processes) are removed, + * short-lived processes (i.e. processes that only live for the + duration of two samples or less) are removed, + * the processes used by the boot logger are removed, + * exploders (i.e. processes that are known to spawn huge meaningless + process subtrees) have their subtrees merged together, + * siblings (i.e. processes with the same command line living + concurrently -- thread heuristic) are merged together, + * process runs (unary trees with processes sharing the command line) + are merged together. + + """ + LOGGER_PROC = 'bootchart-colle' + EXPLODER_PROCESSES = set(['hwup']) + + def __init__(self, writer, kernel, psstats, sample_period, + monitoredApp, prune, idle, taskstats, + accurate_parentage, for_testing = False): + self.writer = writer + self.process_tree = [] + self.taskstats = taskstats + if psstats is None: + process_list = kernel + elif kernel is None: + process_list = psstats.process_map.values() + else: + process_list = list(kernel) + list(psstats.process_map.values()) + self.process_list = sorted(process_list, key = lambda p: p.pid) + self.sample_period = sample_period + + self.build() + if not accurate_parentage: + self.update_ppids_for_daemons(self.process_list) + + self.start_time = self.get_start_time(self.process_tree) + self.end_time = self.get_end_time(self.process_tree) + self.duration = self.end_time - self.start_time + self.idle = idle + + if for_testing: + return + + removed = self.merge_logger(self.process_tree, self.LOGGER_PROC, monitoredApp, False) + writer.status("merged %i logger processes" % removed) + + if prune: + p_processes = self.prune(self.process_tree, None) + p_exploders = self.merge_exploders(self.process_tree, self.EXPLODER_PROCESSES) + p_threads = self.merge_siblings(self.process_tree) + p_runs = self.merge_runs(self.process_tree) + writer.status("pruned %i process, %i exploders, %i threads, and %i runs" % (p_processes, p_exploders, p_threads, p_runs)) + + self.sort(self.process_tree) + + self.start_time = self.get_start_time(self.process_tree) + self.end_time = self.get_end_time(self.process_tree) + self.duration = self.end_time - self.start_time + + self.num_proc = self.num_nodes(self.process_tree) + + def build(self): + """Build the process tree from the list of top samples.""" + self.process_tree = [] + for proc in self.process_list: + if not proc.parent: + self.process_tree.append(proc) + else: + proc.parent.child_list.append(proc) + + def sort(self, process_subtree): + """Sort process tree.""" + for p in process_subtree: + p.child_list.sort(key = lambda p: p.pid) + self.sort(p.child_list) + + def num_nodes(self, process_list): + "Counts the number of nodes in the specified process tree.""" + nodes = 0 + for proc in process_list: + nodes = nodes + self.num_nodes(proc.child_list) + return nodes + len(process_list) + + def get_start_time(self, process_subtree): + """Returns the start time of the process subtree. This is the start + time of the earliest process. + + """ + if not process_subtree: + return 100000000 + return min( [min(proc.start_time, self.get_start_time(proc.child_list)) for proc in process_subtree] ) + + def get_end_time(self, process_subtree): + """Returns the end time of the process subtree. This is the end time + of the last collected sample. + + """ + if not process_subtree: + return -100000000 + return max( [max(proc.start_time + proc.duration, self.get_end_time(proc.child_list)) for proc in process_subtree] ) + + def get_max_pid(self, process_subtree): + """Returns the max PID found in the process tree.""" + if not process_subtree: + return -100000000 + return max( [max(proc.pid, self.get_max_pid(proc.child_list)) for proc in process_subtree] ) + + def update_ppids_for_daemons(self, process_list): + """Fedora hack: when loading the system services from rc, runuser(1) + is used. This sets the PPID of all daemons to 1, skewing + the process tree. Try to detect this and set the PPID of + these processes the PID of rc. + + """ + rcstartpid = -1 + rcendpid = -1 + rcproc = None + for p in process_list: + if p.cmd == "rc" and p.ppid // 1000 == 1: + rcproc = p + rcstartpid = p.pid + rcendpid = self.get_max_pid(p.child_list) + if rcstartpid != -1 and rcendpid != -1: + for p in process_list: + if p.pid > rcstartpid and p.pid < rcendpid and p.ppid // 1000 == 1: + p.ppid = rcstartpid + p.parent = rcproc + for p in process_list: + p.child_list = [] + self.build() + + def prune(self, process_subtree, parent): + """Prunes the process tree by removing idle processes and processes + that only live for the duration of a single top sample. Sibling + processes with the same command line (i.e. threads) are merged + together. This filters out sleepy background processes, short-lived + processes and bootcharts' analysis tools. + """ + def is_idle_background_process_without_children(p): + process_end = p.start_time + p.duration + return not p.active and \ + process_end >= self.start_time + self.duration and \ + p.start_time > self.start_time and \ + p.duration > 0.9 * self.duration and \ + self.num_nodes(p.child_list) == 0 + + num_removed = 0 + idx = 0 + while idx < len(process_subtree): + p = process_subtree[idx] + if parent != None or len(p.child_list) == 0: + + prune = False + if is_idle_background_process_without_children(p): + prune = True + elif p.duration <= 2 * self.sample_period: + # short-lived process + prune = True + + if prune: + process_subtree.pop(idx) + for c in p.child_list: + process_subtree.insert(idx, c) + num_removed += 1 + continue + else: + num_removed += self.prune(p.child_list, p) + else: + num_removed += self.prune(p.child_list, p) + idx += 1 + + return num_removed + + def merge_logger(self, process_subtree, logger_proc, monitored_app, app_tree): + """Merges the logger's process subtree. The logger will typically + spawn lots of sleep and cat processes, thus polluting the + process tree. + + """ + num_removed = 0 + for p in process_subtree: + is_app_tree = app_tree + if logger_proc == p.cmd and not app_tree: + is_app_tree = True + num_removed += self.merge_logger(p.child_list, logger_proc, monitored_app, is_app_tree) + # don't remove the logger itself + continue + + if app_tree and monitored_app != None and monitored_app == p.cmd: + is_app_tree = False + + if is_app_tree: + for child in p.child_list: + self.merge_processes(p, child) + num_removed += 1 + p.child_list = [] + else: + num_removed += self.merge_logger(p.child_list, logger_proc, monitored_app, is_app_tree) + return num_removed + + def merge_exploders(self, process_subtree, processes): + """Merges specific process subtrees (used for processes which usually + spawn huge meaningless process trees). + + """ + num_removed = 0 + for p in process_subtree: + if processes in processes and len(p.child_list) > 0: + subtreemap = self.getProcessMap(p.child_list) + for child in subtreemap.values(): + self.merge_processes(p, child) + num_removed += len(subtreemap) + p.child_list = [] + p.cmd += " (+)" + else: + num_removed += self.merge_exploders(p.child_list, processes) + return num_removed + + def merge_siblings(self, process_subtree): + """Merges thread processes. Sibling processes with the same command + line are merged together. + + """ + num_removed = 0 + idx = 0 + while idx < len(process_subtree)-1: + p = process_subtree[idx] + nextp = process_subtree[idx+1] + if nextp.cmd == p.cmd: + process_subtree.pop(idx+1) + idx -= 1 + num_removed += 1 + p.child_list.extend(nextp.child_list) + self.merge_processes(p, nextp) + num_removed += self.merge_siblings(p.child_list) + idx += 1 + if len(process_subtree) > 0: + p = process_subtree[-1] + num_removed += self.merge_siblings(p.child_list) + return num_removed + + def merge_runs(self, process_subtree): + """Merges process runs. Single child processes which share the same + command line with the parent are merged. + + """ + num_removed = 0 + idx = 0 + while idx < len(process_subtree): + p = process_subtree[idx] + if len(p.child_list) == 1 and p.child_list[0].cmd == p.cmd: + child = p.child_list[0] + p.child_list = list(child.child_list) + self.merge_processes(p, child) + num_removed += 1 + continue + num_removed += self.merge_runs(p.child_list) + idx += 1 + return num_removed + + def merge_processes(self, p1, p2): + """Merges two process' samples.""" + p1.samples.extend(p2.samples) + p1.samples.sort( key = lambda p: p.time ) + p1time = p1.start_time + p2time = p2.start_time + p1.start_time = min(p1time, p2time) + pendtime = max(p1time + p1.duration, p2time + p2.duration) + p1.duration = pendtime - p1.start_time diff --git a/scripts/pybootchartgui/pybootchartgui/samples.py b/scripts/pybootchartgui/pybootchartgui/samples.py new file mode 100644 index 0000000..015d743 --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/samples.py @@ -0,0 +1,151 @@ +# This file is part of pybootchartgui. + +# pybootchartgui is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# pybootchartgui is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with pybootchartgui. If not, see <http://www.gnu.org/licenses/>. + + +class DiskStatSample: + def __init__(self, time): + self.time = time + self.diskdata = [0, 0, 0] + def add_diskdata(self, new_diskdata): + self.diskdata = [ a + b for a, b in zip(self.diskdata, new_diskdata) ] + +class CPUSample: + def __init__(self, time, user, sys, io = 0.0, swap = 0.0): + self.time = time + self.user = user + self.sys = sys + self.io = io + self.swap = swap + + @property + def cpu(self): + return self.user + self.sys + + def __str__(self): + return str(self.time) + "\t" + str(self.user) + "\t" + \ + str(self.sys) + "\t" + str(self.io) + "\t" + str (self.swap) + +class MemSample: + used_values = ('MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree',) + + def __init__(self, time): + self.time = time + self.records = {} + + def add_value(self, name, value): + if name in MemSample.used_values: + self.records[name] = value + + def valid(self): + keys = self.records.keys() + # discard incomplete samples + return [v for v in MemSample.used_values if v not in keys] == [] + +class ProcessSample: + def __init__(self, time, state, cpu_sample): + self.time = time + self.state = state + self.cpu_sample = cpu_sample + + def __str__(self): + return str(self.time) + "\t" + str(self.state) + "\t" + str(self.cpu_sample) + +class ProcessStats: + def __init__(self, writer, process_map, sample_count, sample_period, start_time, end_time): + self.process_map = process_map + self.sample_count = sample_count + self.sample_period = sample_period + self.start_time = start_time + self.end_time = end_time + writer.info ("%d samples, avg. sample length %f" % (self.sample_count, self.sample_period)) + writer.info ("process list size: %d" % len (self.process_map.values())) + +class Process: + def __init__(self, writer, pid, cmd, ppid, start_time): + self.writer = writer + self.pid = pid + self.cmd = cmd + self.exe = cmd + self.args = [] + self.ppid = ppid + self.start_time = start_time + self.duration = 0 + self.samples = [] + self.parent = None + self.child_list = [] + + self.active = None + self.last_user_cpu_time = None + self.last_sys_cpu_time = None + + self.last_cpu_ns = 0 + self.last_blkio_delay_ns = 0 + self.last_swapin_delay_ns = 0 + + # split this process' run - triggered by a name change + def split(self, writer, pid, cmd, ppid, start_time): + split = Process (writer, pid, cmd, ppid, start_time) + + split.last_cpu_ns = self.last_cpu_ns + split.last_blkio_delay_ns = self.last_blkio_delay_ns + split.last_swapin_delay_ns = self.last_swapin_delay_ns + + return split + + def __str__(self): + return " ".join([str(self.pid), self.cmd, str(self.ppid), '[ ' + str(len(self.samples)) + ' samples ]' ]) + + def calc_stats(self, samplePeriod): + if self.samples: + firstSample = self.samples[0] + lastSample = self.samples[-1] + self.start_time = min(firstSample.time, self.start_time) + self.duration = lastSample.time - self.start_time + samplePeriod + + activeCount = sum( [1 for sample in self.samples if sample.cpu_sample and sample.cpu_sample.sys + sample.cpu_sample.user + sample.cpu_sample.io > 0.0] ) + activeCount = activeCount + sum( [1 for sample in self.samples if sample.state == 'D'] ) + self.active = (activeCount>2) + + def calc_load(self, userCpu, sysCpu, interval): + userCpuLoad = float(userCpu - self.last_user_cpu_time) / interval + sysCpuLoad = float(sysCpu - self.last_sys_cpu_time) / interval + cpuLoad = userCpuLoad + sysCpuLoad + # normalize + if cpuLoad > 1.0: + userCpuLoad = userCpuLoad / cpuLoad + sysCpuLoad = sysCpuLoad / cpuLoad + return (userCpuLoad, sysCpuLoad) + + def set_parent(self, processMap): + if self.ppid != None: + self.parent = processMap.get (self.ppid) + if self.parent == None and self.pid // 1000 > 1 and \ + not (self.ppid == 2000 or self.pid == 2000): # kernel threads: ppid=2 + self.writer.warn("Missing CONFIG_PROC_EVENTS: no parent for pid '%i' ('%s') with ppid '%i'" \ + % (self.pid,self.cmd,self.ppid)) + + def get_end_time(self): + return self.start_time + self.duration + +class DiskSample: + def __init__(self, time, read, write, util): + self.time = time + self.read = read + self.write = write + self.util = util + self.tput = read + write + + def __str__(self): + return "\t".join([str(self.time), str(self.read), str(self.write), str(self.util)]) diff --git a/scripts/pybootchartgui/pybootchartgui/tests/parser_test.py b/scripts/pybootchartgui/pybootchartgui/tests/parser_test.py new file mode 100644 index 0000000..00fb3bf --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/tests/parser_test.py @@ -0,0 +1,105 @@ +import sys, os, re, struct, operator, math +from collections import defaultdict +import unittest + +sys.path.insert(0, os.getcwd()) + +import pybootchartgui.parsing as parsing +import pybootchartgui.main as main + +debug = False + +def floatEq(f1, f2): + return math.fabs(f1-f2) < 0.00001 + +bootchart_dir = os.path.join(os.path.dirname(sys.argv[0]), '../../examples/1/') +parser = main._mk_options_parser() +options, args = parser.parse_args(['--q', bootchart_dir]) +writer = main._mk_writer(options) + +class TestBCParser(unittest.TestCase): + + def setUp(self): + self.name = "My first unittest" + self.rootdir = bootchart_dir + + def mk_fname(self,f): + return os.path.join(self.rootdir, f) + + def testParseHeader(self): + trace = parsing.Trace(writer, args, options) + state = parsing.parse_file(writer, trace, self.mk_fname('header')) + self.assertEqual(6, len(state.headers)) + self.assertEqual(2, parsing.get_num_cpus(state.headers)) + + def test_parseTimedBlocks(self): + trace = parsing.Trace(writer, args, options) + state = parsing.parse_file(writer, trace, self.mk_fname('proc_diskstats.log')) + self.assertEqual(141, len(state.disk_stats)) + + def testParseProcPsLog(self): + trace = parsing.Trace(writer, args, options) + state = parsing.parse_file(writer, trace, self.mk_fname('proc_ps.log')) + samples = state.ps_stats + processes = samples.process_map + sorted_processes = [processes[k] for k in sorted(processes.keys())] + + ps_data = open(self.mk_fname('extract2.proc_ps.log')) + for index, line in enumerate(ps_data): + tokens = line.split(); + process = sorted_processes[index] + if debug: + print(tokens[0:4]) + print(process.pid / 1000, process.cmd, process.ppid, len(process.samples)) + print('-------------------') + + self.assertEqual(tokens[0], str(process.pid // 1000)) + self.assertEqual(tokens[1], str(process.cmd)) + self.assertEqual(tokens[2], str(process.ppid // 1000)) + self.assertEqual(tokens[3], str(len(process.samples))) + ps_data.close() + + def testparseProcDiskStatLog(self): + trace = parsing.Trace(writer, args, options) + state_with_headers = parsing.parse_file(writer, trace, self.mk_fname('header')) + state_with_headers.headers['system.cpu'] = 'xxx (2)' + samples = parsing.parse_file(writer, state_with_headers, self.mk_fname('proc_diskstats.log')).disk_stats + self.assertEqual(141, len(samples)) + + diskstats_data = open(self.mk_fname('extract.proc_diskstats.log')) + for index, line in enumerate(diskstats_data): + tokens = line.split('\t') + sample = samples[index] + if debug: + print(line.rstrip()) + print(sample) + print('-------------------') + + self.assertEqual(tokens[0], str(sample.time)) + self.assert_(floatEq(float(tokens[1]), sample.read)) + self.assert_(floatEq(float(tokens[2]), sample.write)) + self.assert_(floatEq(float(tokens[3]), sample.util)) + diskstats_data.close() + + def testparseProcStatLog(self): + trace = parsing.Trace(writer, args, options) + samples = parsing.parse_file(writer, trace, self.mk_fname('proc_stat.log')).cpu_stats + self.assertEqual(141, len(samples)) + + stat_data = open(self.mk_fname('extract.proc_stat.log')) + for index, line in enumerate(stat_data): + tokens = line.split('\t') + sample = samples[index] + if debug: + print(line.rstrip()) + print(sample) + print('-------------------') + self.assert_(floatEq(float(tokens[0]), sample.time)) + self.assert_(floatEq(float(tokens[1]), sample.user)) + self.assert_(floatEq(float(tokens[2]), sample.sys)) + self.assert_(floatEq(float(tokens[3]), sample.io)) + stat_data.close() + +if __name__ == '__main__': + unittest.main() + diff --git a/scripts/pybootchartgui/pybootchartgui/tests/process_tree_test.py b/scripts/pybootchartgui/pybootchartgui/tests/process_tree_test.py new file mode 100644 index 0000000..6f46a1c --- /dev/null +++ b/scripts/pybootchartgui/pybootchartgui/tests/process_tree_test.py @@ -0,0 +1,92 @@ +import sys +import os +import unittest + +sys.path.insert(0, os.getcwd()) + +import pybootchartgui.parsing as parsing +import pybootchartgui.process_tree as process_tree +import pybootchartgui.main as main + +if sys.version_info >= (3, 0): + long = int + +class TestProcessTree(unittest.TestCase): + + def setUp(self): + self.name = "Process tree unittest" + self.rootdir = os.path.join(os.path.dirname(sys.argv[0]), '../../examples/1/') + + parser = main._mk_options_parser() + options, args = parser.parse_args(['--q', self.rootdir]) + writer = main._mk_writer(options) + trace = parsing.Trace(writer, args, options) + + parsing.parse_file(writer, trace, self.mk_fname('proc_ps.log')) + trace.compile(writer) + self.processtree = process_tree.ProcessTree(writer, None, trace.ps_stats, \ + trace.ps_stats.sample_period, None, options.prune, None, None, False, for_testing = True) + + def mk_fname(self,f): + return os.path.join(self.rootdir, f) + + def flatten(self, process_tree): + flattened = [] + for p in process_tree: + flattened.append(p) + flattened.extend(self.flatten(p.child_list)) + return flattened + + def checkAgainstJavaExtract(self, filename, process_tree): + test_data = open(filename) + for expected, actual in zip(test_data, self.flatten(process_tree)): + tokens = expected.split('\t') + self.assertEqual(int(tokens[0]), actual.pid // 1000) + self.assertEqual(tokens[1], actual.cmd) + self.assertEqual(long(tokens[2]), 10 * actual.start_time) + self.assert_(long(tokens[3]) - 10 * actual.duration < 5, "duration") + self.assertEqual(int(tokens[4]), len(actual.child_list)) + self.assertEqual(int(tokens[5]), len(actual.samples)) + test_data.close() + + def testBuild(self): + process_tree = self.processtree.process_tree + self.checkAgainstJavaExtract(self.mk_fname('extract.processtree.1.log'), process_tree) + + def testMergeLogger(self): + self.processtree.merge_logger(self.processtree.process_tree, 'bootchartd', None, False) + process_tree = self.processtree.process_tree + self.checkAgainstJavaExtract(self.mk_fname('extract.processtree.2.log'), process_tree) + + def testPrune(self): + self.processtree.merge_logger(self.processtree.process_tree, 'bootchartd', None, False) + self.processtree.prune(self.processtree.process_tree, None) + process_tree = self.processtree.process_tree + self.checkAgainstJavaExtract(self.mk_fname('extract.processtree.3b.log'), process_tree) + + def testMergeExploders(self): + self.processtree.merge_logger(self.processtree.process_tree, 'bootchartd', None, False) + self.processtree.prune(self.processtree.process_tree, None) + self.processtree.merge_exploders(self.processtree.process_tree, set(['hwup'])) + process_tree = self.processtree.process_tree + self.checkAgainstJavaExtract(self.mk_fname('extract.processtree.3c.log'), process_tree) + + def testMergeSiblings(self): + self.processtree.merge_logger(self.processtree.process_tree, 'bootchartd', None, False) + self.processtree.prune(self.processtree.process_tree, None) + self.processtree.merge_exploders(self.processtree.process_tree, set(['hwup'])) + self.processtree.merge_siblings(self.processtree.process_tree) + process_tree = self.processtree.process_tree + self.checkAgainstJavaExtract(self.mk_fname('extract.processtree.3d.log'), process_tree) + + def testMergeRuns(self): + self.processtree.merge_logger(self.processtree.process_tree, 'bootchartd', None, False) + self.processtree.prune(self.processtree.process_tree, None) + self.processtree.merge_exploders(self.processtree.process_tree, set(['hwup'])) + self.processtree.merge_siblings(self.processtree.process_tree) + self.processtree.merge_runs(self.processtree.process_tree) + process_tree = self.processtree.process_tree + self.checkAgainstJavaExtract(self.mk_fname('extract.processtree.3e.log'), process_tree) + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/pythondeps b/scripts/pythondeps new file mode 100755 index 0000000..ff92e74 --- /dev/null +++ b/scripts/pythondeps @@ -0,0 +1,250 @@ +#!/usr/bin/env python +# +# Determine dependencies of python scripts or available python modules in a search path. +# +# Given the -d argument and a filename/filenames, returns the modules imported by those files. +# Given the -d argument and a directory/directories, recurses to find all +# python packages and modules, returns the modules imported by these. +# Given the -p argument and a path or paths, scans that path for available python modules/packages. + +import argparse +import ast +import imp +import logging +import os.path +import sys + + +logger = logging.getLogger('pythondeps') + +suffixes = [] +for triple in imp.get_suffixes(): + suffixes.append(triple[0]) + + +class PythonDepError(Exception): + pass + + +class DependError(PythonDepError): + def __init__(self, path, error): + self.path = path + self.error = error + PythonDepError.__init__(self, error) + + def __str__(self): + return "Failure determining dependencies of {}: {}".format(self.path, self.error) + + +class ImportVisitor(ast.NodeVisitor): + def __init__(self): + self.imports = set() + self.importsfrom = [] + + def visit_Import(self, node): + for alias in node.names: + self.imports.add(alias.name) + + def visit_ImportFrom(self, node): + self.importsfrom.append((node.module, [a.name for a in node.names], node.level)) + + +def walk_up(path): + while path: + yield path + path, _, _ = path.rpartition(os.sep) + + +def get_provides(path): + path = os.path.realpath(path) + + def get_fn_name(fn): + for suffix in suffixes: + if fn.endswith(suffix): + return fn[:-len(suffix)] + + isdir = os.path.isdir(path) + if isdir: + pkg_path = path + walk_path = path + else: + pkg_path = get_fn_name(path) + if pkg_path is None: + return + walk_path = os.path.dirname(path) + + for curpath in walk_up(walk_path): + if not os.path.exists(os.path.join(curpath, '__init__.py')): + libdir = curpath + break + else: + libdir = '' + + package_relpath = pkg_path[len(libdir)+1:] + package = '.'.join(package_relpath.split(os.sep)) + if not isdir: + yield package, path + else: + if os.path.exists(os.path.join(path, '__init__.py')): + yield package, path + + for dirpath, dirnames, filenames in os.walk(path): + relpath = dirpath[len(path)+1:] + if relpath: + if '__init__.py' not in filenames: + dirnames[:] = [] + continue + else: + context = '.'.join(relpath.split(os.sep)) + if package: + context = package + '.' + context + yield context, dirpath + else: + context = package + + for fn in filenames: + adjusted_fn = get_fn_name(fn) + if not adjusted_fn or adjusted_fn == '__init__': + continue + + fullfn = os.path.join(dirpath, fn) + if context: + yield context + '.' + adjusted_fn, fullfn + else: + yield adjusted_fn, fullfn + + +def get_code_depends(code_string, path=None, provide=None, ispkg=False): + try: + code = ast.parse(code_string, path) + except TypeError as exc: + raise DependError(path, exc) + except SyntaxError as exc: + raise DependError(path, exc) + + visitor = ImportVisitor() + visitor.visit(code) + for builtin_module in sys.builtin_module_names: + if builtin_module in visitor.imports: + visitor.imports.remove(builtin_module) + + if provide: + provide_elements = provide.split('.') + if ispkg: + provide_elements.append("__self__") + context = '.'.join(provide_elements[:-1]) + package_path = os.path.dirname(path) + else: + context = None + package_path = None + + levelzero_importsfrom = (module for module, names, level in visitor.importsfrom + if level == 0) + for module in visitor.imports | set(levelzero_importsfrom): + if context and path: + module_basepath = os.path.join(package_path, module.replace('.', '/')) + if os.path.exists(module_basepath): + # Implicit relative import + yield context + '.' + module, path + continue + + for suffix in suffixes: + if os.path.exists(module_basepath + suffix): + # Implicit relative import + yield context + '.' + module, path + break + else: + yield module, path + else: + yield module, path + + for module, names, level in visitor.importsfrom: + if level == 0: + continue + elif not provide: + raise DependError("Error: ImportFrom non-zero level outside of a package: {0}".format((module, names, level)), path) + elif level > len(provide_elements): + raise DependError("Error: ImportFrom level exceeds package depth: {0}".format((module, names, level)), path) + else: + context = '.'.join(provide_elements[:-level]) + if module: + if context: + yield context + '.' + module, path + else: + yield module, path + + +def get_file_depends(path): + try: + code_string = open(path, 'r').read() + except (OSError, IOError) as exc: + raise DependError(path, exc) + + return get_code_depends(code_string, path) + + +def get_depends_recursive(directory): + directory = os.path.realpath(directory) + + provides = dict((v, k) for k, v in get_provides(directory)) + for filename, provide in provides.iteritems(): + if os.path.isdir(filename): + filename = os.path.join(filename, '__init__.py') + ispkg = True + elif not filename.endswith('.py'): + continue + else: + ispkg = False + + with open(filename, 'r') as f: + source = f.read() + + depends = get_code_depends(source, filename, provide, ispkg) + for depend, by in depends: + yield depend, by + + +def get_depends(path): + if os.path.isdir(path): + return get_depends_recursive(path) + else: + return get_file_depends(path) + + +def main(): + logging.basicConfig() + + parser = argparse.ArgumentParser(description='Determine dependencies and provided packages for python scripts/modules') + parser.add_argument('path', nargs='+', help='full path to content to be processed') + group = parser.add_mutually_exclusive_group() + group.add_argument('-p', '--provides', action='store_true', + help='given a path, display the provided python modules') + group.add_argument('-d', '--depends', action='store_true', + help='given a filename, display the imported python modules') + + args = parser.parse_args() + if args.provides: + modules = set() + for path in args.path: + for provide, fn in get_provides(path): + modules.add(provide) + + for module in sorted(modules): + print(module) + elif args.depends: + for path in args.path: + try: + modules = get_depends(path) + except PythonDepError as exc: + logger.error(str(exc)) + sys.exit(1) + + for module, imp_by in modules: + print("{}\t{}".format(module, imp_by)) + else: + parser.print_help() + sys.exit(2) + + +if __name__ == '__main__': + main() diff --git a/scripts/recipetool b/scripts/recipetool new file mode 100755 index 0000000..6c66487 --- /dev/null +++ b/scripts/recipetool @@ -0,0 +1,121 @@ +#!/usr/bin/env python + +# Recipe creation tool +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import os +import argparse +import glob +import logging + +scripts_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] +import scriptutils +import argparse_oe +logger = scriptutils.logger_create('recipetool') + +plugins = [] + +def tinfoil_init(parserecipes): + import bb.tinfoil + import logging + tinfoil = bb.tinfoil.Tinfoil(tracking=True) + tinfoil.prepare(not parserecipes) + tinfoil.logger.setLevel(logger.getEffectiveLevel()) + return tinfoil + +def main(): + + if not os.environ.get('BUILDDIR', ''): + logger.error("This script can only be run after initialising the build environment (e.g. by using oe-init-build-env)") + sys.exit(1) + + parser = argparse_oe.ArgumentParser(description="OpenEmbedded recipe tool", + add_help=False, + epilog="Use %(prog)s <subcommand> --help to get help on a specific command") + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') + parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR') + + global_args, unparsed_args = parser.parse_known_args() + + # Help is added here rather than via add_help=True, as we don't want it to + # be handled by parse_known_args() + parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, + help='show this help message and exit') + subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') + + if global_args.debug: + logger.setLevel(logging.DEBUG) + elif global_args.quiet: + logger.setLevel(logging.ERROR) + + import scriptpath + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + sys.exit(1) + logger.debug('Found bitbake path: %s' % bitbakepath) + + scriptutils.logger_setup_color(logger, global_args.color) + + tinfoil = tinfoil_init(False) + for path in ([scripts_path] + + tinfoil.config_data.getVar('BBPATH', True).split(':')): + pluginpath = os.path.join(path, 'lib', 'recipetool') + scriptutils.load_plugins(logger, plugins, pluginpath) + + registered = False + for plugin in plugins: + if hasattr(plugin, 'register_commands'): + registered = True + plugin.register_commands(subparsers) + elif hasattr(plugin, 'register_command'): + # Legacy function name + registered = True + plugin.register_command(subparsers) + if hasattr(plugin, 'tinfoil_init'): + plugin.tinfoil_init(tinfoil) + + if not registered: + logger.error("No commands registered - missing plugins?") + sys.exit(1) + + args = parser.parse_args(unparsed_args, namespace=global_args) + + try: + if getattr(args, 'parserecipes', False): + tinfoil.config_data.disableTracking() + tinfoil.parseRecipes() + tinfoil.config_data.enableTracking() + ret = args.func(args) + except bb.BBHandledException: + ret = 1 + + return ret + + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/scripts/relocate_sdk.py b/scripts/relocate_sdk.py new file mode 100755 index 0000000..99fca86 --- /dev/null +++ b/scripts/relocate_sdk.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python +# +# Copyright (c) 2012 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# DESCRIPTION +# This script is called by the SDK installer script. It replaces the dynamic +# loader path in all binaries and also fixes the SYSDIR paths/lengths and the +# location of ld.so.cache in the dynamic loader binary +# +# AUTHORS +# Laurentiu Palcu <laurentiu.palcu@intel.com> +# + +import struct +import sys +import stat +import os +import re +import errno + +if sys.version < '3': + def b(x): + return x +else: + def b(x): + return x.encode(sys.getfilesystemencoding()) + +old_prefix = re.compile(b("##DEFAULT_INSTALL_DIR##")) + +def get_arch(): + f.seek(0) + e_ident =f.read(16) + ei_mag0,ei_mag1_3,ei_class = struct.unpack("<B3sB11x", e_ident) + + if (ei_mag0 != 0x7f and ei_mag1_3 != "ELF") or ei_class == 0: + return 0 + + if ei_class == 1: + return 32 + elif ei_class == 2: + return 64 + +def parse_elf_header(): + global e_type, e_machine, e_version, e_entry, e_phoff, e_shoff, e_flags,\ + e_ehsize, e_phentsize, e_phnum, e_shentsize, e_shnum, e_shstrndx + + f.seek(0) + elf_header = f.read(64) + + if arch == 32: + # 32bit + hdr_fmt = "<HHILLLIHHHHHH" + hdr_size = 52 + else: + # 64bit + hdr_fmt = "<HHIQQQIHHHHHH" + hdr_size = 64 + + e_type, e_machine, e_version, e_entry, e_phoff, e_shoff, e_flags,\ + e_ehsize, e_phentsize, e_phnum, e_shentsize, e_shnum, e_shstrndx =\ + struct.unpack(hdr_fmt, elf_header[16:hdr_size]) + +def change_interpreter(elf_file_name): + if arch == 32: + ph_fmt = "<IIIIIIII" + else: + ph_fmt = "<IIQQQQQQ" + + """ look for PT_INTERP section """ + for i in range(0,e_phnum): + f.seek(e_phoff + i * e_phentsize) + ph_hdr = f.read(e_phentsize) + if arch == 32: + # 32bit + p_type, p_offset, p_vaddr, p_paddr, p_filesz,\ + p_memsz, p_flags, p_align = struct.unpack(ph_fmt, ph_hdr) + else: + # 64bit + p_type, p_flags, p_offset, p_vaddr, p_paddr, \ + p_filesz, p_memsz, p_align = struct.unpack(ph_fmt, ph_hdr) + + """ change interpreter """ + if p_type == 3: + # PT_INTERP section + f.seek(p_offset) + # External SDKs with mixed pre-compiled binaries should not get + # relocated so look for some variant of /lib + fname = f.read(11) + if fname.startswith(b("/lib/")) or fname.startswith(b("/lib64/")) or \ + fname.startswith(b("/lib32/")) or fname.startswith(b("/usr/lib32/")) or \ + fname.startswith(b("/usr/lib32/")) or fname.startswith(b("/usr/lib64/")): + break + if (len(new_dl_path) >= p_filesz): + print("ERROR: could not relocate %s, interp size = %i and %i is needed." \ + % (elf_file_name, p_memsz, len(new_dl_path) + 1)) + break + dl_path = new_dl_path + b("\0") * (p_filesz - len(new_dl_path)) + f.seek(p_offset) + f.write(dl_path) + break + +def change_dl_sysdirs(elf_file_name): + if arch == 32: + sh_fmt = "<IIIIIIIIII" + else: + sh_fmt = "<IIQQQQIIQQ" + + """ read section string table """ + f.seek(e_shoff + e_shstrndx * e_shentsize) + sh_hdr = f.read(e_shentsize) + if arch == 32: + sh_offset, sh_size = struct.unpack("<16xII16x", sh_hdr) + else: + sh_offset, sh_size = struct.unpack("<24xQQ24x", sh_hdr) + + f.seek(sh_offset) + sh_strtab = f.read(sh_size) + + sysdirs = sysdirs_len = "" + + """ change ld.so.cache path and default libs path for dynamic loader """ + for i in range(0,e_shnum): + f.seek(e_shoff + i * e_shentsize) + sh_hdr = f.read(e_shentsize) + + sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link,\ + sh_info, sh_addralign, sh_entsize = struct.unpack(sh_fmt, sh_hdr) + + name = sh_strtab[sh_name:sh_strtab.find(b("\0"), sh_name)] + + """ look only into SHT_PROGBITS sections """ + if sh_type == 1: + f.seek(sh_offset) + """ default library paths cannot be changed on the fly because """ + """ the string lengths have to be changed too. """ + if name == b(".sysdirs"): + sysdirs = f.read(sh_size) + sysdirs_off = sh_offset + sysdirs_sect_size = sh_size + elif name == b(".sysdirslen"): + sysdirslen = f.read(sh_size) + sysdirslen_off = sh_offset + elif name == b(".ldsocache"): + ldsocache_path = f.read(sh_size) + new_ldsocache_path = old_prefix.sub(new_prefix, ldsocache_path) + new_ldsocache_path = new_ldsocache_path.rstrip(b("\0")) + if (len(new_ldsocache_path) >= sh_size): + print("ERROR: could not relocate %s, .ldsocache section size = %i and %i is needed." \ + % (elf_file_name, sh_size, len(new_ldsocache_path))) + sys.exit(-1) + # pad with zeros + new_ldsocache_path += b("\0") * (sh_size - len(new_ldsocache_path)) + # write it back + f.seek(sh_offset) + f.write(new_ldsocache_path) + elif name == b(".gccrelocprefix"): + offset = 0 + while (offset + 4096) <= sh_size: + path = f.read(4096) + new_path = old_prefix.sub(new_prefix, path) + new_path = new_path.rstrip(b("\0")) + if (len(new_path) >= 4096): + print("ERROR: could not relocate %s, max path size = 4096 and %i is needed." \ + % (elf_file_name, len(new_path))) + sys.exit(-1) + # pad with zeros + new_path += b("\0") * (4096 - len(new_path)) + #print "Changing %s to %s at %s" % (str(path), str(new_path), str(offset)) + # write it back + f.seek(sh_offset + offset) + f.write(new_path) + offset = offset + 4096 + if sysdirs != "" and sysdirslen != "": + paths = sysdirs.split(b("\0")) + sysdirs = b("") + sysdirslen = b("") + for path in paths: + """ exit the loop when we encounter first empty string """ + if path == b(""): + break + + new_path = old_prefix.sub(new_prefix, path) + sysdirs += new_path + b("\0") + + if arch == 32: + sysdirslen += struct.pack("<L", len(new_path)) + else: + sysdirslen += struct.pack("<Q", len(new_path)) + + """ pad with zeros """ + sysdirs += b("\0") * (sysdirs_sect_size - len(sysdirs)) + + """ write the sections back """ + f.seek(sysdirs_off) + f.write(sysdirs) + f.seek(sysdirslen_off) + f.write(sysdirslen) + +# MAIN +if len(sys.argv) < 4: + sys.exit(-1) + +# In python > 3, strings may also contain Unicode characters. So, convert +# them to bytes +if sys.version_info < (3,): + new_prefix = sys.argv[1] + new_dl_path = sys.argv[2] +else: + new_prefix = sys.argv[1].encode() + new_dl_path = sys.argv[2].encode() + +executables_list = sys.argv[3:] + +for e in executables_list: + perms = os.stat(e)[stat.ST_MODE] + if os.access(e, os.W_OK|os.R_OK): + perms = None + else: + os.chmod(e, perms|stat.S_IRWXU) + + try: + f = open(e, "r+b") + except IOError: + exctype, ioex = sys.exc_info()[:2] + if ioex.errno == errno.ETXTBSY: + print("Could not open %s. File used by another process.\nPlease "\ + "make sure you exit all processes that might use any SDK "\ + "binaries." % e) + else: + print("Could not open %s: %s(%d)" % (e, ioex.strerror, ioex.errno)) + sys.exit(-1) + + # Save old size and do a size check at the end. Just a safety measure. + old_size = os.path.getsize(e) + if old_size >= 64: + arch = get_arch() + if arch: + parse_elf_header() + change_interpreter(e) + change_dl_sysdirs(e) + + """ change permissions back """ + if perms: + os.chmod(e, perms) + + f.close() + + if old_size != os.path.getsize(e): + print("New file size for %s is different. Looks like a relocation error!", e) + sys.exit(-1) + diff --git a/scripts/rootfs_rpm-extract-postinst.awk b/scripts/rootfs_rpm-extract-postinst.awk new file mode 100644 index 0000000..8f2836b --- /dev/null +++ b/scripts/rootfs_rpm-extract-postinst.awk @@ -0,0 +1,11 @@ +/Name:.*/ { + package = substr($0, 7) + next +} +/postinstall.*scriptlet .*/ { + next +} +{ + print $0 >> ENVIRON["D"] "/etc/rpm-postinsts/" package ".sh" +} + diff --git a/scripts/rpm2cpio.sh b/scripts/rpm2cpio.sh new file mode 100755 index 0000000..5df8c0f --- /dev/null +++ b/scripts/rpm2cpio.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +# This comes from the RPM5 5.4.0 distribution. + +pkg=$1 +if [ "$pkg" = "" -o ! -e "$pkg" ]; then + echo "no package supplied" 1>&2 + exit 1 +fi + +leadsize=96 +o=`expr $leadsize + 8` +set `od -j $o -N 8 -t u1 $pkg` +il=`expr 256 \* \( 256 \* \( 256 \* $2 + $3 \) + $4 \) + $5` +dl=`expr 256 \* \( 256 \* \( 256 \* $6 + $7 \) + $8 \) + $9` +# echo "sig il: $il dl: $dl" + +sigsize=`expr 8 + 16 \* $il + $dl` +o=`expr $o + $sigsize + \( 8 - \( $sigsize \% 8 \) \) \% 8 + 8` +set `od -j $o -N 8 -t u1 $pkg` +il=`expr 256 \* \( 256 \* \( 256 \* $2 + $3 \) + $4 \) + $5` +dl=`expr 256 \* \( 256 \* \( 256 \* $6 + $7 \) + $8 \) + $9` +# echo "hdr il: $il dl: $dl" + +hdrsize=`expr 8 + 16 \* $il + $dl` +o=`expr $o + $hdrsize` +EXTRACTOR="dd if=$pkg ibs=$o skip=1" + +COMPRESSION=`($EXTRACTOR |file -) 2>/dev/null` +if echo $COMPRESSION |grep -iq gzip; then + DECOMPRESSOR=gunzip +elif echo $COMPRESSION |grep -iq bzip2; then + DECOMPRESSOR=bunzip2 +elif echo $COMPRESSION |grep -iq xz; then + DECOMPRESSOR=unxz +elif echo $COMPRESSION |grep -iq cpio; then + DECOMPRESSOR=cat +else + # Most versions of file don't support LZMA, therefore we assume + # anything not detected is LZMA + DECOMPRESSOR=`which unlzma 2>/dev/null` + case "$DECOMPRESSOR" in + /* ) ;; + * ) DECOMPRESSOR=`which lzmash 2>/dev/null` + case "$DECOMPRESSOR" in + /* ) DECOMPRESSOR="lzmash -d -c" ;; + * ) DECOMPRESSOR=cat ;; + esac + ;; + esac +fi + +$EXTRACTOR 2>/dev/null | $DECOMPRESSOR diff --git a/scripts/runqemu b/scripts/runqemu new file mode 100755 index 0000000..d7fa941 --- /dev/null +++ b/scripts/runqemu @@ -0,0 +1,546 @@ +#!/bin/bash +# +# Handle running OE images standalone with QEMU +# +# Copyright (C) 2006-2011 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +usage() { + MYNAME=`basename $0` +cat <<_EOF + +Usage: you can run this script with any valid combination +of the following environment variables (in any order): + KERNEL - the kernel image file to use + ROOTFS - the rootfs image file or nfsroot directory to use + MACHINE - the machine name (optional, autodetected from KERNEL filename if unspecified) + Simplified QEMU command-line options can be passed with: + nographic - disables video console + serial - enables a serial console on /dev/ttyS0 + kvm - enables KVM when running qemux86/qemux86-64 (VT-capable CPU required) + kvm-vhost - enables KVM with vhost support when running qemux86/qemux86-64 (VT-capable CPU required) + publicvnc - enable a VNC server open to all hosts + qemuparams="xyz" - specify custom parameters to QEMU + bootparams="xyz" - specify custom kernel parameters during boot + +Examples: + $MYNAME qemuarm + $MYNAME qemux86-64 core-image-sato ext4 + $MYNAME qemux86-64 wic-image-minimal wic + $MYNAME path/to/bzImage-qemux86.bin path/to/nfsrootdir/ serial + $MYNAME qemux86 iso/hddimg/vmdk/qcow2/vdi/ramfs/cpio.gz... + $MYNAME qemux86 qemuparams="-m 256" + $MYNAME qemux86 bootparams="psplash=false" + $MYNAME path/to/<image>-<machine>.vmdk + $MYNAME path/to/<image>-<machine>.wic +_EOF + exit 1 +} + +if [ "x$1" = "x" ]; then + usage +fi + +error() { + echo "Error: "$* + usage +} + +MACHINE=${MACHINE:=""} +KERNEL=${KERNEL:=""} +ROOTFS=${ROOTFS:=""} +FSTYPE=${FSTYPE:=""} +LAZY_ROOTFS="" +SCRIPT_QEMU_OPT="" +SCRIPT_QEMU_EXTRA_OPT="" +SCRIPT_KERNEL_OPT="" +SERIALSTDIO="" +TCPSERIAL_PORTNUM="" +KVM_ENABLED="no" +KVM_ACTIVE="no" +VHOST_ENABLED="no" +VHOST_ACTIVE="no" +IS_VM="false" + +# Determine whether the file is a kernel or QEMU image, and set the +# appropriate variables +process_filename() { + filename=$1 + + # Extract the filename extension + EXT=`echo $filename | awk -F . '{ print \$NF }'` + case /$EXT/ in + /bin/) + # A file ending in .bin is a kernel + [ -z "$KERNEL" ] && KERNEL=$filename || \ + error "conflicting KERNEL args [$KERNEL] and [$filename]" + ;; + /ext[234]/|/jffs2/|/btrfs/) + # A file ending in a supportted fs type is a rootfs image + if [ -z "$FSTYPE" -o "$FSTYPE" = "$EXT" ]; then + FSTYPE=$EXT + ROOTFS=$filename + else + error "conflicting FSTYPE types [$FSTYPE] and [$EXT]" + fi + ;; + /hddimg/|/hdddirect/|/vmdk/|/wic/|/qcow2/|/vdi/) + FSTYPE=$EXT + VM=$filename + ROOTFS=$filename + IS_VM="true" + ;; + *) + error "unknown file arg [$filename]" + ;; + esac +} + +check_fstype_conflicts() { + if [ -z "$FSTYPE" -o "$FSTYPE" = "$1" ]; then + FSTYPE=$1 + else + error "conflicting FSTYPE types [$FSTYPE] and [$1]" + fi +} +# Parse command line args without requiring specific ordering. It's a +# bit more complex, but offers a great user experience. +while true; do + arg=${1} + case "$arg" in + "qemux86" | "qemux86-64" | "qemuarm" | "qemuarm64" | "qemumips" | "qemumipsel" | \ + "qemumips64" | "qemush4" | "qemuppc" | "qemumicroblaze" | "qemuzynq") + [ -z "$MACHINE" -o "$MACHINE" = "$arg" ] && MACHINE=$arg || \ + error "conflicting MACHINE types [$MACHINE] and [$arg]" + ;; + "ext"[234] | "jffs2" | "nfs" | "btrfs") + check_fstype_conflicts $arg + ;; + "hddimg" | "hdddirect" | "wic" | "vmdk" | "qcow2" | "vdi" | "iso") + check_fstype_conflicts $arg + IS_VM="true" + ;; + "ramfs" | "cpio.gz") + FSTYPE=cpio.gz + ;; + "nographic") + SCRIPT_QEMU_OPT="$SCRIPT_QEMU_OPT -nographic" + SCRIPT_KERNEL_OPT="$SCRIPT_KERNEL_OPT console=ttyS0" + ;; + "serial") + SCRIPT_QEMU_OPT="$SCRIPT_QEMU_OPT -serial stdio" + SCRIPT_KERNEL_OPT="$SCRIPT_KERNEL_OPT console=ttyS0" + SERIALSTDIO="1" + ;; + "tcpserial="*) + TCPSERIAL_PORTNUM=${arg##tcpserial=} + ;; + "biosdir="*) + CUSTOMBIOSDIR="${arg##biosdir=}" + ;; + "biosfilename="*) + SCRIPT_QEMU_OPT="$SCRIPT_QEMU_OPT -bios ${arg##biosfilename=}" + ;; + "qemuparams="*) + SCRIPT_QEMU_EXTRA_OPT="${arg##qemuparams=}" + + # Warn user if they try to specify serial or kvm options + # to use simplified options instead + serial_option=`expr "$SCRIPT_QEMU_EXTRA_OPT" : '.*\(-serial\)'` + kvm_option=`expr "$SCRIPT_QEMU_EXTRA_OPT" : '.*\(-enable-kvm\)'` + vga_option=`expr "$SCRIPT_QEMU_EXTRA_OPT" : '.*\(-vga\)'` + [ ! -z "$serial_option" -o ! -z "$kvm_option" ] && \ + echo "Please use simplified serial or kvm options instead" + ;; + "bootparams="*) + SCRIPT_KERNEL_OPT="$SCRIPT_KERNEL_OPT ${arg##bootparams=}" + ;; + "audio") + if [ "x$MACHINE" = "xqemux86" -o "x$MACHINE" = "xqemux86-64" ]; then + echo "Enabling audio in qemu." + echo "Please install snd_intel8x0 or snd_ens1370 driver in linux guest." + QEMU_AUDIO_DRV="alsa" + SCRIPT_QEMU_OPT="$SCRIPT_QEMU_OPT -soundhw ac97,es1370" + fi + ;; + "kvm") + KVM_ENABLED="yes" + KVM_CAPABLE=`grep -q 'vmx\|svm' /proc/cpuinfo && echo 1` + ;; + "kvm-vhost") + KVM_ENABLED="yes" + KVM_CAPABLE=`grep -q 'vmx\|svm' /proc/cpuinfo && echo 1` + VHOST_ENABLED="yes" + ;; + "slirp") + SLIRP_ENABLED="yes" + ;; + "publicvnc") + SCRIPT_QEMU_OPT="$SCRIPT_QEMU_OPT -vnc :0" + ;; + *-image*) + [ -z "$ROOTFS" ] || \ + error "conflicting ROOTFS args [$ROOTFS] and [$arg]" + if [ -f "$arg" ]; then + process_filename $arg + elif [ -d "$arg" ]; then + # Handle the case where the nfsroot dir has -image- + # in the pathname + echo "Assuming $arg is an nfs rootfs" + FSTYPE=nfs + ROOTFS=$arg + else + ROOTFS=$arg + LAZY_ROOTFS="true" + fi + ;; + "") break ;; + *) + # A directory name is an nfs rootfs + if [ -d "$arg" ]; then + echo "Assuming $arg is an nfs rootfs" + if [ -z "$FSTYPE" -o "$FSTYPE" = "nfs" ]; then + FSTYPE=nfs + else + error "conflicting FSTYPE types [$arg] and nfs" + fi + + if [ -z "$ROOTFS" ]; then + ROOTFS=$arg + else + error "conflicting ROOTFS args [$ROOTFS] and [$arg]" + fi + elif [ -f "$arg" ]; then + process_filename $arg + else + error "unable to classify arg [$arg]" + fi + ;; + esac + shift +done + +if [ ! -c /dev/net/tun ] ; then + echo "TUN control device /dev/net/tun is unavailable; you may need to enable TUN (e.g. sudo modprobe tun)" + exit 1 +elif [ ! -w /dev/net/tun ] ; then + echo "TUN control device /dev/net/tun is not writable, please fix (e.g. sudo chmod 666 /dev/net/tun)" + exit 1 +fi + +# Report errors for missing combinations of options +if [ -z "$MACHINE" -a -z "$KERNEL" -a -z "$VM" -a "$FSTYPE" != "wic" ]; then + error "you must specify at least a MACHINE or KERNEL argument" +fi +if [ "$FSTYPE" = "nfs" -a -z "$ROOTFS" ]; then + error "NFS booting without an explicit ROOTFS path is not yet supported" +fi + +if [ -z "$MACHINE" ]; then + if [ "$IS_VM" = "true" ]; then + [ "x$FSTYPE" = "xwic" ] && filename=$ROOTFS || filename=$VM + MACHINE=`basename $filename | sed -n 's/.*\(qemux86-64\|qemux86\|qemuarm64\|qemuarm\|qemumips64\|qemumips\|qemuppc\|qemush4\).*/\1/p'` + if [ -z "$MACHINE" ]; then + error "Unable to set MACHINE from image filename [$VM]" + fi + echo "Set MACHINE to [$MACHINE] based on image [$VM]" + else + MACHINE=`basename $KERNEL | sed -n 's/.*\(qemux86-64\|qemux86\|qemuarm64\|qemuarm\|qemumips64\|qemumips\|qemuppc\|qemush4\).*/\1/p'` + if [ -z "$MACHINE" ]; then + error "Unable to set MACHINE from kernel filename [$KERNEL]" + fi + echo "Set MACHINE to [$MACHINE] based on kernel [$KERNEL]" + fi +fi + +YOCTO_KVM_WIKI="https://wiki.yoctoproject.org/wiki/How_to_enable_KVM_for_Poky_qemu" +YOCTO_PARAVIRT_KVM_WIKI="https://wiki.yoctoproject.org/wiki/Running_an_x86_Yocto_Linux_image_under_QEMU_KVM" +# Detect KVM configuration +if [ "x$KVM_ENABLED" = "xyes" ]; then + if [ -z "$KVM_CAPABLE" ]; then + echo "You are trying to enable KVM on a cpu without VT support." + echo "Remove kvm from the command-line, or refer" + echo "$YOCTO_KVM_WIKI"; + exit 1; + fi + if [ "x$MACHINE" != "xqemux86" -a "x$MACHINE" != "xqemux86-64" ]; then + echo "KVM only support x86 & x86-64. Remove kvm from the command-line"; + exit 1; + fi + if [ ! -e /dev/kvm ]; then + echo "Missing KVM device. Have you inserted kvm modules?" + echo "For further help see:" + echo "$YOCTO_KVM_WIKI"; + exit 1; + fi + if [ -w /dev/kvm -a -r /dev/kvm ]; then + SCRIPT_QEMU_OPT="$SCRIPT_QEMU_OPT -enable-kvm" + KVM_ACTIVE="yes" + else + echo "You have no rights on /dev/kvm." + echo "Please change the ownership of this file as described at:" + echo "$YOCTO_KVM_WIKI"; + exit 1; + fi + if [ "x$VHOST_ENABLED" = "xyes" ]; then + if [ ! -e /dev/vhost-net ]; then + echo "Missing virtio net device. Have you inserted vhost-net module?" + echo "For further help see:" + echo "$YOCTO_PARAVIRT_KVM_WIKI"; + exit 1; + fi + + if [ -w /dev/vhost-net -a -r /dev/vhost-net ]; then + VHOST_ACTIVE="yes" + else + echo "You have no rights on /dev/vhost-net." + echo "Please change the ownership of this file as described at:" + echo "$YOCTO_KVM_WIKI"; + exit 1; + fi + fi +fi + +machine2=`echo $MACHINE | tr 'a-z' 'A-Z' | sed 's/-/_/'` +# MACHINE is now set for all cases + +# Defaults used when these vars need to be inferred +QEMUX86_DEFAULT_KERNEL=bzImage-qemux86.bin +QEMUX86_DEFAULT_FSTYPE=ext4 + +QEMUX86_64_DEFAULT_KERNEL=bzImage-qemux86-64.bin +QEMUX86_64_DEFAULT_FSTYPE=ext4 + +QEMUARM_DEFAULT_KERNEL=zImage-qemuarm.bin +QEMUARM_DEFAULT_FSTYPE=ext4 + +QEMUARM64_DEFAULT_KERNEL=Image-qemuarm64.bin +QEMUARM64_DEFAULT_FSTYPE=ext4 + +QEMUMIPS_DEFAULT_KERNEL=vmlinux-qemumips.bin +QEMUMIPS_DEFAULT_FSTYPE=ext4 + +QEMUMIPSEL_DEFAULT_KERNEL=vmlinux-qemumipsel.bin +QEMUMIPSEL_DEFAULT_FSTYPE=ext4 + +QEMUMIPS64_DEFAULT_KERNEL=vmlinux-qemumips64.bin +QEMUMIPS64_DEFAULT_FSTYPE=ext4 + +QEMUSH4_DEFAULT_KERNEL=vmlinux-qemumips.bin +QEMUSH4_DEFAULT_FSTYPE=ext4 + +QEMUPPC_DEFAULT_KERNEL=vmlinux-qemuppc.bin +QEMUPPC_DEFAULT_FSTYPE=ext4 + +QEMUMICROBLAZE_DEFAULT_KERNEL=linux.bin.ub +QEMUMICROBLAZE_DEFAULT_FSTYPE=cpio + +QEMUZYNQ_DEFAULT_KERNEL=uImage +QEMUZYNQ_DEFAULT_FSTYPE=cpio + +setup_path_vars() { + if [ -z "$OE_TMPDIR" ] ; then + PATHS_REQUIRED=true + elif [ "$1" = "1" -a -z "$DEPLOY_DIR_IMAGE" ] ; then + PATHS_REQUIRED=true + else + PATHS_REQUIRED=false + fi + + if [ "$PATHS_REQUIRED" = "true" ]; then + # Try to get the variable values from bitbake + type -P bitbake &>/dev/null || { + echo "In order for this script to dynamically infer paths"; + echo "to kernels or filesystem images, you either need"; + echo "bitbake in your PATH or to source oe-init-build-env"; + echo "before running this script" >&2; + exit 1; } + + # We have bitbake in PATH, get the variable values from bitbake + BITBAKE_ENV_TMPFILE=`mktemp --tmpdir runqemu.XXXXXXXXXX` + if [ "$?" != "0" ] ; then + echo "Error: mktemp failed for bitbake environment output" + exit 1 + fi + + MACHINE=$MACHINE bitbake -e > $BITBAKE_ENV_TMPFILE + if [ -z "$OE_TMPDIR" ] ; then + OE_TMPDIR=`sed -n 's/^TMPDIR=\"\(.*\)\"/\1/p' $BITBAKE_ENV_TMPFILE` + fi + if [ -z "$DEPLOY_DIR_IMAGE" ] ; then + DEPLOY_DIR_IMAGE=`sed -n 's/^DEPLOY_DIR_IMAGE=\"\(.*\)\"/\1/p' $BITBAKE_ENV_TMPFILE` + fi + if [ -z "$OE_TMPDIR" ]; then + # Check for errors from bitbake that the user needs to know about + BITBAKE_OUTPUT=`cat $BITBAKE_ENV_TMPFILE | wc -l` + if [ "$BITBAKE_OUTPUT" -eq "0" ]; then + echo "Error: this script needs to be run from your build directory, or you need" + echo "to explicitly set OE_TMPDIR and DEPLOY_DIR_IMAGE in your environment" + else + echo "There was an error running bitbake to determine TMPDIR" + echo "Here is the output from 'bitbake -e':" + cat $BITBAKE_ENV_TMPFILE + fi + rm $BITBAKE_ENV_TMPFILE + exit 1 + fi + rm $BITBAKE_ENV_TMPFILE + fi +} + +setup_sysroot() { + # Toolchain installs set up $OECORE_NATIVE_SYSROOT in their + # environment script. If that variable isn't set, we're + # either in an in-tree build scenario or the environment + # script wasn't source'd. + if [ -z "$OECORE_NATIVE_SYSROOT" ]; then + setup_path_vars + BUILD_ARCH=`uname -m` + BUILD_OS=`uname | tr '[A-Z]' '[a-z]'` + BUILD_SYS="$BUILD_ARCH-$BUILD_OS" + + OECORE_NATIVE_SYSROOT=$OE_TMPDIR/sysroots/$BUILD_SYS + fi + + # Some recipes store the BIOS under $OE_TMPDIR/sysroots/$MACHINE, + # now defined as OECORE_MACHINE_SYSROOT. The latter is used when searching + # BIOS, VGA BIOS and keymaps. + if [ -z "$OECORE_MACHINE_SYSROOT" ]; then + OECORE_MACHINE_SYSROOT=$OE_TMPDIR/sysroots/$MACHINE + fi +} + +# Locate a rootfs image to boot which matches our expected +# machine and fstype. +findimage() { + where=$1 + machine=$2 + extension=$3 + + # Sort rootfs candidates by modification time - the most + # recently created one is the one we most likely want to boot. + filename=`ls -t1 $where/*-image*$machine.$extension 2>/dev/null | head -n1` + if [ "x$filename" != "x" ]; then + ROOTFS=$filename + return + fi + + echo "Couldn't find a $machine rootfs image in $where." + exit 1 +} + +if [ -e "$ROOTFS" -a -z "$FSTYPE" ]; then + # Extract the filename extension + EXT=`echo $ROOTFS | awk -F . '{ print \$NF }'` + if [ "x$EXT" = "xext2" -o "x$EXT" = "xext3" -o \ + "x$EXT" = "xjffs2" -o "x$EXT" = "xbtrfs" -o \ + "x$EXT" = "xext4" ]; then + FSTYPE=$EXT + else + echo "Note: Unable to determine filesystem extension for $ROOTFS" + echo "We will use the default FSTYPE for $MACHINE" + # ...which is done further below... + fi +fi + +if [ -z "$KERNEL" -a "$IS_VM" = "false" ]; then \ + setup_path_vars 1 + eval kernel_file=\$${machine2}_DEFAULT_KERNEL + KERNEL=$DEPLOY_DIR_IMAGE/$kernel_file + + if [ -z "$KERNEL" ]; then + error "Unable to determine default kernel for MACHINE [$MACHINE]" + fi +fi +# KERNEL is now set for all cases + +if [ -z "$FSTYPE" ]; then + eval FSTYPE=\$${machine2}_DEFAULT_FSTYPE + + if [ -z "$FSTYPE" ]; then + error "Unable to determine default fstype for MACHINE [$MACHINE]" + fi +fi + +# FSTYPE is now set for all cases + +# Handle cases where a ROOTFS type is given instead of a filename, e.g. +# core-image-sato +if [ "$LAZY_ROOTFS" = "true" ]; then + setup_path_vars 1 + echo "Assuming $ROOTFS really means $DEPLOY_DIR_IMAGE/$ROOTFS-$MACHINE.$FSTYPE" + if [ "$IS_VM" = "true" ]; then + VM=$DEPLOY_DIR_IMAGE/$ROOTFS-$MACHINE.$FSTYPE + else + ROOTFS=$DEPLOY_DIR_IMAGE/$ROOTFS-$MACHINE.$FSTYPE + fi +fi + +if [ -z "$ROOTFS" ]; then + setup_path_vars 1 + T=$DEPLOY_DIR_IMAGE + eval rootfs_list=\$${machine2}_DEFAULT_ROOTFS + findimage $T $MACHINE $FSTYPE + + if [ -z "$ROOTFS" ]; then + error "Unable to determine default rootfs for MACHINE [$MACHINE]" + elif [ "$IS_VM" = "true" ]; then + VM=$ROOTFS + fi +fi +# ROOTFS is now set for all cases, now expand it to be an absolute path, it should exist at this point + +ROOTFS=`readlink -f $ROOTFS` + +echo "" +echo "Continuing with the following parameters:" +if [ "$IS_VM" = "false" ]; then + echo "KERNEL: [$KERNEL]" + echo "ROOTFS: [$ROOTFS]" +else + echo "VM: [$VM]" +fi +echo "FSTYPE: [$FSTYPE]" + +setup_sysroot +# OECORE_NATIVE_SYSROOT and OECORE_MACHINE_SYSROOT are now set for all cases + +INTERNAL_SCRIPT="$0-internal" +if [ ! -f "$INTERNAL_SCRIPT" -o ! -r "$INTERNAL_SCRIPT" ]; then +INTERNAL_SCRIPT=`which runqemu-internal` +fi + +# Specify directory for BIOS, VGA BIOS and keymaps +if [ ! -z "$CUSTOMBIOSDIR" ]; then + if [ -d "$OECORE_NATIVE_SYSROOT/$CUSTOMBIOSDIR" ]; then + echo "Assuming biosdir is $OECORE_NATIVE_SYSROOT/$CUSTOMBIOSDIR" + SCRIPT_QEMU_OPT="$SCRIPT_QEMU_OPT -L $OECORE_NATIVE_SYSROOT/$CUSTOMBIOSDIR" + elif [ -d "$OECORE_MACHINE_SYSROOT/$CUSTOMBIOSDIR" ]; then + echo "Assuming biosdir is $OECORE_MACHINE_SYSROOT/$CUSTOMBIOSDIR" + SCRIPT_QEMU_OPT="$SCRIPT_QEMU_OPT -L $OECORE_MACHINE_SYSROOT/$CUSTOMBIOSDIR" + else + if [ ! -d "$CUSTOMBIOSDIR" ]; then + echo "Custom BIOS directory not found. Tried: $CUSTOMBIOSDIR" + echo "and $OECORE_NATIVE_SYSROOT/$CUSTOMBIOSDIR" + echo "and $OECORE_MACHINE_SYSROOT/$CUSTOMBIOSDIR" + exit 1; + fi + echo "Assuming biosdir is $CUSTOMBIOSDIR" + SCRIPT_QEMU_OPT="$SCRIPT_QEMU_OPT -L $CUSTOMBIOSDIR" + fi +fi + +. $INTERNAL_SCRIPT +exit $? diff --git a/scripts/runqemu-addptable2image b/scripts/runqemu-addptable2image new file mode 100755 index 0000000..f0195ad --- /dev/null +++ b/scripts/runqemu-addptable2image @@ -0,0 +1,51 @@ +#!/bin/sh + +# Add a partion table to an ext2 image file +# +# Copyright (C) 2006-2007 OpenedHand Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +IMAGE=$1 +IMAGEOUT=$2 + +echo $IMAGE +echo $IMAGEOUT + +size=`ls -l $IMAGE | awk '{ print $5}'` +size2=`expr $size / 512 / 16 / 63` + +echo $size +echo $size2 + +# MBR Size = 512 * 63 bytes +dd if=/dev/zero of=$IMAGEOUT count=63 + +echo "x" > /tmp/fdisk.cmds +echo "c" >> /tmp/fdisk.cmds +echo "1024" >> /tmp/fdisk.cmds +echo "h" >> /tmp/fdisk.cmds +echo "16" >> /tmp/fdisk.cmds +echo "r" >> /tmp/fdisk.cmds +echo "n" >> /tmp/fdisk.cmds +echo "p" >> /tmp/fdisk.cmds +echo "1" >> /tmp/fdisk.cmds +echo "1" >> /tmp/fdisk.cmds +echo "$size2" >> /tmp/fdisk.cmds +echo "w" >> /tmp/fdisk.cmds + +/sbin/fdisk $IMAGEOUT < /tmp/fdisk.cmds +cat $IMAGE >> $IMAGEOUT diff --git a/scripts/runqemu-export-rootfs b/scripts/runqemu-export-rootfs new file mode 100755 index 0000000..3dee131 --- /dev/null +++ b/scripts/runqemu-export-rootfs @@ -0,0 +1,163 @@ +#!/bin/bash +# +# Copyright (c) 2005-2009 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +usage() { + echo "Usage: $0 {start|stop|restart} <nfs-export-dir>" +} + +if [ $# != 2 ]; then + usage + exit 1 +fi + +if [[ "$1" != "start" && "$1" != "stop" && "$1" != "restart" ]]; then + echo "Unknown command '$1'" + usage + exit 1 +fi + +if [ ! -d "$2" ]; then + echo "Error: '$2' does not exist" + usage + exit 1 +fi +# Ensure the nfs-export-dir is an absolute path +NFS_EXPORT_DIR=$(cd "$2" && pwd) + +SYSROOT_SETUP_SCRIPT=`which oe-find-native-sysroot 2> /dev/null` +if [ -z "$SYSROOT_SETUP_SCRIPT" ]; then + echo "Error: Unable to find the oe-find-native-sysroot script" + echo "Did you forget to source your build environment setup script?" + exit 1 +fi +. $SYSROOT_SETUP_SCRIPT + +if [ ! -e "$OECORE_NATIVE_SYSROOT/usr/bin/unfsd" ]; then + echo "Error: Unable to find unfsd binary in $OECORE_NATIVE_SYSROOT/usr/bin/" + + if [ "x$OECORE_DISTRO_VERSION" = "x" ]; then + echo "Have you run 'bitbake meta-ide-support'?" + else + echo "This shouldn't happen - something is missing from your toolchain installation" + fi + exit 1 +fi + +if [ ! -d ~/.runqemu-sdk ]; then + mkdir -p ~/.runqemu-sdk +fi + +NFS_INSTANCE=${NFS_INSTANCE:=0} +EXPORTS=~/.runqemu-sdk/exports$NFS_INSTANCE +RMTAB=~/.runqemu-sdk/rmtab$NFS_INSTANCE +NFSPID=~/.runqemu-sdk/nfs$NFS_INSTANCE.pid +MOUNTPID=~/.runqemu-sdk/mount$NFS_INSTANCE.pid + +PSEUDO_OPTS="-P $OECORE_NATIVE_SYSROOT/usr" +PSEUDO_LOCALSTATEDIR="$NFS_EXPORT_DIR/../$(basename $NFS_EXPORT_DIR).pseudo_state" +export PSEUDO_LOCALSTATEDIR + +if [ ! -d "$PSEUDO_LOCALSTATEDIR" ]; then + echo "Error: $PSEUDO_LOCALSTATEDIR does not exist." + echo "Did you create the export directory using runqemu-extract-sdk?" + exit 1 +fi + +# rpc.mountd RPC port +NFS_MOUNTPROG=$[ 21111 + $NFS_INSTANCE ] +# rpc.nfsd RPC port +NFS_NFSPROG=$[ 11111 + $NFS_INSTANCE ] +# NFS port number +NFS_PORT=$[ 3049 + 2 * $NFS_INSTANCE ] +# mountd port number +MOUNT_PORT=$[ 3048 + 2 * $NFS_INSTANCE ] + +## For debugging you would additionally add +## --debug all +UNFSD_OPTS="-p -N -i $NFSPID -e $EXPORTS -x $NFS_NFSPROG -n $NFS_PORT -y $NFS_MOUNTPROG -m $MOUNT_PORT" + +# Setup the exports file +if [ "$1" = "start" ]; then + echo "Creating exports file..." + echo "$NFS_EXPORT_DIR (rw,async,no_root_squash,no_all_squash,insecure)" > $EXPORTS +fi + +# See how we were called. +case "$1" in + start) + PORTMAP_RUNNING=`ps -ef | grep portmap | grep -v grep` + RPCBIND_RUNNING=`ps -ef | grep rpcbind | grep -v grep` + if [[ "x$PORTMAP_RUNNING" = "x" && "x$RPCBIND_RUNNING" = "x" ]]; then + echo "=======================================================" + echo "Error: neither rpcbind nor portmap appear to be running" + echo "Please install and start one of these services first" + echo "=======================================================" + echo "Tip: for recent Ubuntu hosts, run:" + echo " sudo apt-get install rpcbind" + echo "Then add OPTIONS=\"-i -w\" to /etc/default/rpcbind and run" + echo " sudo service portmap restart" + + exit 1 + fi + + echo "Starting User Mode nfsd" + echo " $PSEUDO $PSEUDO_OPTS $OECORE_NATIVE_SYSROOT/usr/bin/unfsd $UNFSD_OPTS" + $PSEUDO $PSEUDO_OPTS $OECORE_NATIVE_SYSROOT/usr/bin/unfsd $UNFSD_OPTS + if [ ! $? = 0 ]; then + echo "Error starting nfsd" + exit 1 + fi + # Check to make sure everything started ok. + if [ ! -f $NFSPID ]; then + echo "rpc.nfsd did not start correctly" + exit 1 + fi + ps -fp `cat $NFSPID` > /dev/null 2> /dev/null + if [ ! $? = 0 ]; then + echo "rpc.nfsd did not start correctly" + exit 1 + fi + echo " " + echo "On your target please remember to add the following options for NFS" + echo "nfsroot=IP_ADDRESS:$NFS_EXPORT_DIR,nfsvers=3,port=$NFSD_PORT,mountprog=$MOUNTD_RPCPORT,nfsprog=$NFSD_RPCPORT,udp,mountport=$MOUNTD_PORT" + ;; + stop) + if [ -f "$NFSPID" ]; then + echo "Stopping rpc.nfsd" + kill `cat $NFSPID` + rm -f $NFSPID + else + echo "No PID file, not stopping rpc.nfsd" + fi + if [ -f "$EXPORTS" ]; then + echo "Removing exports file" + rm -f $EXPORTS + fi + ;; + restart) + $0 stop $NFS_EXPORT_DIR + $0 start $NFS_EXPORT_DIR + if [ ! $? = 0 ]; then + exit 1 + fi + ;; + *) + echo "$0 {start|stop|restart} <nfs-export-dir>" + ;; +esac + +exit 0 diff --git a/scripts/runqemu-extract-sdk b/scripts/runqemu-extract-sdk new file mode 100755 index 0000000..32ddd48 --- /dev/null +++ b/scripts/runqemu-extract-sdk @@ -0,0 +1,104 @@ +#!/bin/bash +# +# This utility extracts an SDK image tarball using pseudo, and stores +# the pseudo database in var/pseudo within the rootfs. If you want to +# boot QEMU using an nfsroot, you *must* use this script to create the +# rootfs to ensure it is done correctly with pseudo. +# +# Copyright (c) 2010 Intel Corp. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +function usage() { + echo "Usage: $0 <image-tarball> <extract-dir>" +} + +if [ $# -ne 2 ]; then + usage + exit 1 +fi + +SYSROOT_SETUP_SCRIPT=`which oe-find-native-sysroot 2> /dev/null` +if [ -z "$SYSROOT_SETUP_SCRIPT" ]; then + echo "Error: Unable to find the oe-find-native-sysroot script" + echo "Did you forget to source your build system environment setup script?" + exit 1 +fi +. $SYSROOT_SETUP_SCRIPT +PSEUDO_OPTS="-P $OECORE_NATIVE_SYSROOT/usr" + +ROOTFS_TARBALL=$1 +SDK_ROOTFS_DIR=$2 + +if [ ! -e "$ROOTFS_TARBALL" ]; then + echo "Error: sdk tarball '$ROOTFS_TARBALL' does not exist" + usage + exit 1 +fi + +# Convert SDK_ROOTFS_DIR to a full pathname +if [[ ${SDK_ROOTFS_DIR:0:1} != "/" ]]; then + SDK_ROOTFS_DIR=$(readlink -f $(pwd)/$SDK_ROOTFS_DIR) +fi + +TAR_OPTS="" +if [[ "$ROOTFS_TARBALL" =~ tar\.bz2$ ]]; then + TAR_OPTS="--numeric-owner -xjf" +fi +if [[ "$ROOTFS_TARBALL" =~ tar\.gz$ ]]; then + TAR_OPTS="--numeric-owner -xzf" +fi +if [[ "$ROOTFS_TARBALL" =~ \.tar$ ]]; then + TAR_OPTS="--numeric-owner -xf" +fi +if [ -z "$TAR_OPTS" ]; then + echo "Error: Unable to determine sdk tarball format" + echo "Accepted types: .tar / .tar.gz / .tar.bz2" + exit 1 +fi + +if [ ! -d "$SDK_ROOTFS_DIR" ]; then + echo "Creating directory $SDK_ROOTFS_DIR" + mkdir -p "$SDK_ROOTFS_DIR" +fi + +pseudo_state_dir="$SDK_ROOTFS_DIR/../$(basename "$SDK_ROOTFS_DIR").pseudo_state" +pseudo_state_dir="$(readlink -f $pseudo_state_dir)" + +if [ -e "$pseudo_state_dir" ]; then + echo "Error: $pseudo_state_dir already exists!" + echo "Please delete the rootfs tree and pseudo directory manually" + echo "if this is really what you want." + exit 1 +fi + +mkdir -p "$pseudo_state_dir" +touch "$pseudo_state_dir/pseudo.pid" +PSEUDO_LOCALSTATEDIR="$pseudo_state_dir" +export PSEUDO_LOCALSTATEDIR + +echo "Extracting rootfs tarball using pseudo..." +echo "$PSEUDO $PSEUDO_OPTS tar -C \"$SDK_ROOTFS_DIR\" $TAR_OPTS \"$ROOTFS_TARBALL\"" +$PSEUDO $PSEUDO_OPTS tar -C "$SDK_ROOTFS_DIR" $TAR_OPTS "$ROOTFS_TARBALL" + +DIRCHECK=`ls -l "$SDK_ROOTFS_DIR" | wc -l` +if [ "$DIRCHECK" -lt 5 ]; then + echo "Warning: I don't see many files in $SDK_ROOTFS_DIR" + echo "Please double-check the extraction worked as intended" + exit 0 +fi + +echo "SDK image successfully extracted to $SDK_ROOTFS_DIR" + +exit 0 diff --git a/scripts/runqemu-gen-tapdevs b/scripts/runqemu-gen-tapdevs new file mode 100755 index 0000000..624deac --- /dev/null +++ b/scripts/runqemu-gen-tapdevs @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Create a "bank" of tap network devices that can be used by the +# runqemu script. This script needs to be run as root, and will +# use the tunctl binary from the build system sysroot. Note: many Linux +# distros these days still use an older version of tunctl which does not +# support the group permissions option, hence the need to use the build +# system provided version. +# +# Copyright (C) 2010 Intel Corp. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +usage() { + echo "Usage: sudo $0 <uid> <gid> <num> <native-sysroot-basedir>" + echo "Where <uid> is the numeric user id the tap devices will be owned by" + echo "Where <gid> is the numeric group id the tap devices will be owned by" + echo "<num> is the number of tap devices to create (0 to remove all)" + echo "<native-sysroot-basedir> is the path to the build system's native sysroot" + exit 1 +} + +if [ $EUID -ne 0 ]; then + echo "Error: This script must be run with root privileges" + exit +fi + +if [ $# -ne 4 ]; then + echo "Error: Incorrect number of arguments" + usage +fi + +TUID=$1 +GID=$2 +COUNT=$3 +SYSROOT=$4 + +TUNCTL=$SYSROOT/usr/bin/tunctl +if [[ ! -x "$TUNCTL" || -d "$TUNCTL" ]]; then + echo "Error: $TUNCTL is not an executable" + usage +fi + +SCRIPT_DIR=`dirname $0` +RUNQEMU_IFUP="$SCRIPT_DIR/runqemu-ifup" +if [ ! -x "$RUNQEMU_IFUP" ]; then + echo "Error: Unable to find the runqemu-ifup script in $SCRIPT_DIR" + exit 1 +fi + +IFCONFIG=`which ip 2> /dev/null` +if [ -z "$IFCONFIG" ]; then + # Is it ever anywhere else? + IFCONFIG=/sbin/ip +fi +if [ ! -x "$IFCONFIG" ]; then + echo "$IFCONFIG cannot be executed" + exit 1 +fi + +# Ensure we start with a clean slate +for tap in `$IFCONFIG link | grep tap | awk '{ print \$2 }' | sed s/://`; do + echo "Note: Destroying pre-existing tap interface $tap..." + $TUNCTL -d $tap +done + +echo "Creating $COUNT tap devices for UID: $TUID GID: $GID..." +for ((index=0; index < $COUNT; index++)); do + echo "Creating tap$index" + ifup=`$RUNQEMU_IFUP $TUID $GID $SYSROOT 2>&1` + if [ $? -ne 0 ]; then + echo "Error running tunctl: $ifup" + exit 1 + fi +done + +if [ $COUNT -gt 0 ]; then + echo "Note: For systems running NetworkManager, it's recommended" + echo "Note: that the tap devices be set as unmanaged in the" + echo "Note: NetworkManager.conf file. Add the following lines to" + echo "Note: /etc/NetworkManager/NetworkManager.conf" + echo "[keyfile]" + echo "unmanaged-devices=interface-name:tap*" +fi + +# The runqemu script will check for this file, and if it exists, +# will use the existing bank of tap devices without creating +# additional ones via sudo. +touch /etc/runqemu-nosudo diff --git a/scripts/runqemu-ifdown b/scripts/runqemu-ifdown new file mode 100755 index 0000000..8f66cfa --- /dev/null +++ b/scripts/runqemu-ifdown @@ -0,0 +1,66 @@ +#!/bin/bash +# +# QEMU network configuration script to bring down tap devices. This +# utility needs to be run as root, and will use the tunctl binary +# from the native sysroot. +# +# If you find yourself calling this script a lot, you can add the +# the following to your /etc/sudoers file to be able to run this +# command without entering your password each time: +# +# <my-username> ALL=NOPASSWD: /path/to/runqemu-ifup +# <my-username> ALL=NOPASSWD: /path/to/runqemu-ifdown +# +# Copyright (c) 2006-2011 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +usage() { + echo "sudo $(basename $0) <tap-dev> <native-sysroot-basedir>" +} + +if [ $EUID -ne 0 ]; then + echo "Error: This script (runqemu-ifdown) must be run with root privileges" + exit 1 +fi + +if [ $# -ne 2 ]; then + usage + exit 1 +fi + +TAP=$1 +NATIVE_SYSROOT_DIR=$2 + +TUNCTL=$NATIVE_SYSROOT_DIR/usr/bin/tunctl +if [ ! -e "$TUNCTL" ]; then + echo "Error: Unable to find tunctl binary in '$NATIVE_SYSROOT_DIR/usr/bin', please bitbake qemu-helper-native" + exit 1 +fi + +$TUNCTL -d $TAP + +# cleanup the remaining iptables rules +IPTABLES=`which iptables 2> /dev/null` +if [ "x$IPTABLES" = "x" ]; then + IPTABLES=/sbin/iptables +fi +if [ ! -x "$IPTABLES" ]; then + echo "$IPTABLES cannot be executed" + exit 1 +fi +n=$[ (`echo $TAP | sed 's/tap//'` * 2) + 1 ] +dest=$[ (`echo $TAP | sed 's/tap//'` * 2) + 2 ] +$IPTABLES -D POSTROUTING -t nat -j MASQUERADE -s 192.168.7.$n/32 +$IPTABLES -D POSTROUTING -t nat -j MASQUERADE -s 192.168.7.$dest/32 diff --git a/scripts/runqemu-ifup b/scripts/runqemu-ifup new file mode 100755 index 0000000..d9bd894 --- /dev/null +++ b/scripts/runqemu-ifup @@ -0,0 +1,121 @@ +#!/bin/bash +# +# QEMU network interface configuration script. This utility needs to +# be run as root, and will use the tunctl binary from a native sysroot. +# Note: many Linux distros these days still use an older version of +# tunctl which does not support the group permissions option, hence +# the need to use build system's version. +# +# If you find yourself calling this script a lot, you can add the +# the following to your /etc/sudoers file to be able to run this +# command without entering your password each time: +# +# <my-username> ALL=NOPASSWD: /path/to/runqemu-ifup +# <my-username> ALL=NOPASSWD: /path/to/runqemu-ifdown +# +# If you'd like to create a bank of tap devices at once, you should use +# the runqemu-gen-tapdevs script instead. If tap devices are set up using +# that script, the runqemu script will never end up calling this +# script. +# +# Copyright (c) 2006-2011 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +usage() { + echo "sudo $(basename $0) <uid> <gid> <native-sysroot-basedir>" +} + +if [ $EUID -ne 0 ]; then + echo "Error: This script (runqemu-ifup) must be run with root privileges" + exit 1 +fi + +if [ $# -ne 3 ]; then + usage + exit 1 +fi + +USERID="-u $1" +GROUP="-g $2" +NATIVE_SYSROOT_DIR=$3 + +TUNCTL=$NATIVE_SYSROOT_DIR/usr/bin/tunctl +if [ ! -x "$TUNCTL" ]; then + echo "Error: Unable to find tunctl binary in '$NATIVE_SYSROOT_DIR/usr/bin', please bitbake qemu-helper-native" + exit 1 +fi + +TAP=`$TUNCTL -b $GROUP 2>&1` +STATUS=$? +if [ $STATUS -ne 0 ]; then +# If tunctl -g fails, try using tunctl -u, for older host kernels +# which do not support the TUNSETGROUP ioctl + TAP=`$TUNCTL -b $USERID 2>&1` + STATUS=$? + if [ $STATUS -ne 0 ]; then + echo "tunctl failed:" + exit 1 + fi +fi + +IFCONFIG=`which ip 2> /dev/null` +if [ "x$IFCONFIG" = "x" ]; then + # better than nothing... + IFCONFIG=/sbin/ip +fi +if [ ! -x "$IFCONFIG" ]; then + echo "$IFCONFIG cannot be executed" + exit 1 +fi + +IPTABLES=`which iptables 2> /dev/null` +if [ "x$IPTABLES" = "x" ]; then + IPTABLES=/sbin/iptables +fi +if [ ! -x "$IPTABLES" ]; then + echo "$IPTABLES cannot be executed" + exit 1 +fi + +n=$[ (`echo $TAP | sed 's/tap//'` * 2) + 1 ] +$IFCONFIG addr add 192.168.7.$n/32 broadcast 192.168.7.255 dev $TAP +STATUS=$? +if [ $STATUS -ne 0 ]; then + echo "Failed to set up IP addressing on $TAP" + exit 1 +fi +$IFCONFIG link set dev $TAP up +STATUS=$? +if [ $STATUS -ne 0 ]; then + echo "Failed to bring up $TAP" + exit 1 +fi + +dest=$[ (`echo $TAP | sed 's/tap//'` * 2) + 2 ] +$IFCONFIG route add to 192.168.7.$dest dev $TAP +STATUS=$? +if [ $STATUS -ne 0 ]; then + echo "Failed to add route to 192.168.7.$dest using $TAP" + exit 1 +fi + +# setup NAT for tap0 interface to have internet access in QEMU +$IPTABLES -A POSTROUTING -t nat -j MASQUERADE -s 192.168.7.$n/32 +$IPTABLES -A POSTROUTING -t nat -j MASQUERADE -s 192.168.7.$dest/32 +echo 1 > /proc/sys/net/ipv4/ip_forward +echo 1 > /proc/sys/net/ipv4/conf/$TAP/proxy_arp +$IPTABLES -P FORWARD ACCEPT + +echo $TAP diff --git a/scripts/runqemu-internal b/scripts/runqemu-internal new file mode 100755 index 0000000..ac1c703 --- /dev/null +++ b/scripts/runqemu-internal @@ -0,0 +1,717 @@ +#!/bin/bash -x + +# Handle running OE images under qemu +# +# Copyright (C) 2006-2011 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Call setting: +# QEMU_MEMORY (optional) - set the amount of memory in the emualted system. +# SERIAL_LOGFILE (optional) - log the serial port output to a file +# +# Image options: +# MACHINE - the machine to run +# FSTYPE - the image type to run +# KERNEL - the kernel image file to use +# ROOTFS - the disk image file to use +# + +mem_size=-1 + +#Get rid of <> and get the contents of extra qemu running params +SCRIPT_QEMU_EXTRA_OPT=`echo $SCRIPT_QEMU_EXTRA_OPT | sed -e 's/<//' -e 's/>//'` +#if user set qemu memory, eg: -m 256 in qemu extra params, we need to do some +# validation check +mem_set=`expr "$SCRIPT_QEMU_EXTRA_OPT" : '.*\(-m[[:space:]] *[0-9]*\)'` +if [ ! -z "$mem_set" ] ; then +#Get memory setting size from user input + mem_size=`echo $mem_set | sed 's/-m[[:space:]] *//'` +fi + +# This file is created when runqemu-gen-tapdevs creates a bank of tap +# devices, indicating that the user should not bring up new ones using +# sudo. +NOSUDO_FLAG="/etc/runqemu-nosudo" + +QEMUIFUP=`which runqemu-ifup 2> /dev/null` +QEMUIFDOWN=`which runqemu-ifdown 2> /dev/null` +if [ -z "$QEMUIFUP" -o ! -x "$QEMUIFUP" ]; then + echo "runqemu-ifup cannot be found or executed" + exit 1 +fi +if [ -z "$QEMUIFDOWN" -o ! -x "$QEMUIFDOWN" ]; then + echo "runqemu-ifdown cannot be found or executed" + exit 1 +fi + +NFSRUNNING="false" + +#capture original stty values +ORIG_STTY=$(stty -g) + +if [ "$SLIRP_ENABLED" = "yes" ]; then + KERNEL_NETWORK_CMD="ip=dhcp" + QEMU_TAP_CMD="" + QEMU_UI_OPTIONS="-show-cursor -usb -usbdevice tablet" + QEMU_NETWORK_CMD="" + DROOT="/dev/vda" + ROOTFS_OPTIONS="-drive file=$ROOTFS,if=virtio,format=raw" +else + acquire_lock() { + lockfile=$1 + if [ -z "$lockfile" ]; then + echo "Error: missing lockfile arg passed to acquire_lock()" + return 1 + fi + + touch $lockfile.lock 2>/dev/null + if [ $? -ne 0 ]; then + echo "Acquiring lockfile for $lockfile.lock failed" + return 1 + fi + exec 8>$lockfile.lock + flock -n -x 8 + if [ $? -ne 0 ]; then + exec 8>&- + return 1 + fi + + return 0 + } + + release_lock() { + lockfile=$1 + if [ -z "$lockfile" ]; then + echo "Error: missing lockfile arg passed to release_lock()" + return 1 + fi + + rm -f $lockfile.lock + exec 8>&- + } + + LOCKDIR="/tmp/qemu-tap-locks" + if [ ! -d "$LOCKDIR" ]; then + mkdir $LOCKDIR + chmod 777 $LOCKDIR + fi + + IFCONFIG=`which ip 2> /dev/null` + if [ -z "$IFCONFIG" ]; then + IFCONFIG=/sbin/ip + fi + if [ ! -x "$IFCONFIG" ]; then + echo "$IFCONFIG cannot be executed" + exit 1 + fi + + POSSIBLE=`$IFCONFIG link | grep 'tap' | awk '{print $2}' | sed -e 's/://' -e 's/@.*//'` + TAP="" + LOCKFILE="" + USE_PRECONF_TAP="no" + for tap in $POSSIBLE; do + LOCKFILE="$LOCKDIR/$tap" + if [ -e "$LOCKFILE.skip" ]; then + echo "Found $LOCKFILE.skip, skipping $tap" + continue + fi + echo "Acquiring lockfile for $tap..." + acquire_lock $LOCKFILE + if [ $? -eq 0 ]; then + TAP=$tap + USE_PRECONF_TAP="yes" + break + fi + done + + if [ "$TAP" = "" ]; then + if [ -e "$NOSUDO_FLAG" ]; then + echo "Error: There are no available tap devices to use for networking," + echo "and I see $NOSUDO_FLAG exists, so I am not going to try creating" + echo "a new one with sudo." + exit 1 + fi + + GROUPID=`id -g` + USERID=`id -u` + echo "Setting up tap interface under sudo" + # Redirect stderr since we could see a LD_PRELOAD warning here if pseudo is loaded + # but inactive. This looks scary but is harmless + tap=`sudo $QEMUIFUP $USERID $GROUPID $OECORE_NATIVE_SYSROOT 2> /dev/null` + if [ $? -ne 0 ]; then + # Re-run standalone to see verbose errors + sudo $QEMUIFUP $USERID $GROUPID $OECORE_NATIVE_SYSROOT + return 1 + fi + LOCKFILE="$LOCKDIR/$tap" + echo "Acquiring lockfile for $tap..." + acquire_lock $LOCKFILE + if [ $? -eq 0 ]; then + TAP=$tap + fi + else + echo "Using preconfigured tap device '$TAP'" + echo "If this is not intended, touch $LOCKFILE.skip to make runqemu skip $TAP." + fi + + cleanup() { + if [ ! -e "$NOSUDO_FLAG" -a "$USE_PRECONF_TAP" = "no" ]; then + # Redirect stderr since we could see a LD_PRELOAD warning here if pseudo is loaded + # but inactive. This looks scary but is harmless + sudo $QEMUIFDOWN $TAP $OECORE_NATIVE_SYSROOT 2> /dev/null + fi + echo "Releasing lockfile of preconfigured tap device '$TAP'" + release_lock $LOCKFILE + + if [ "$NFSRUNNING" = "true" ]; then + echo "Shutting down the userspace NFS server..." + echo "runqemu-export-rootfs stop $ROOTFS" + runqemu-export-rootfs stop $ROOTFS + fi + # If QEMU crashes or somehow tty properties are not restored + # after qemu exits, we need to run stty sane + #stty sane + + #instead of using stty sane we set the original stty values + stty ${ORIG_STTY} + + } + + + n0=$(echo $TAP | sed 's/tap//') + + case $n0 in + ''|*[!0-9]*) + echo "Error Couldn't turn $TAP into an interface number?" + exit 1 + ;; + esac + + n1=$(($n0 * 2 + 1)) + n2=$(($n1 + 1)) + + KERNEL_NETWORK_CMD="ip=192.168.7.$n2::192.168.7.$n1:255.255.255.0" + QEMU_TAP_CMD="-net tap,vlan=0,ifname=$TAP,script=no,downscript=no" + if [ "$VHOST_ACTIVE" = "yes" ]; then + QEMU_NETWORK_CMD="-net nic,model=virtio $QEMU_TAP_CMD,vhost=on" + else + QEMU_NETWORK_CMD="-net nic,model=virtio $QEMU_TAP_CMD" + fi + DROOT="/dev/vda" + ROOTFS_OPTIONS="-drive file=$ROOTFS,if=virtio,format=raw" + + KERNCMDLINE="mem=$QEMU_MEMORY" + QEMU_UI_OPTIONS="-show-cursor -usb -usbdevice tablet" + + NFS_INSTANCE=`echo $TAP | sed 's/tap//'` + export NFS_INSTANCE + + SERIALOPTS="" + if [ "x$SERIAL_LOGFILE" != "x" ]; then + SERIALOPTS="-serial file:$SERIAL_LOGFILE" + fi +fi + +if [ ! -f "$KERNEL" -a "$IS_VM" = "false" ]; then + echo "Error: Kernel image file $KERNEL doesn't exist" + cleanup + return 1 +fi + +if [ "$FSTYPE" != "nfs" -a "$IS_VM" = "false" -a ! -f "$ROOTFS" ]; then + echo "Error: Image file $ROOTFS doesn't exist" + cleanup + return 1 +fi + +if [ "$NFS_SERVER" = "" ]; then + NFS_SERVER="192.168.7.1" + if [ "$SLIRP_ENABLED" = "yes" ]; then + NFS_SERVER="10.0.2.2" + fi +fi + +if [ "$FSTYPE" = "nfs" ]; then + NFS_DIR=`echo $ROOTFS | sed 's/^[^:]*:\(.*\)/\1/'` + if [ "$NFS_INSTANCE" = "" ] ; then + NFS_INSTANCE=0 + fi + MOUNTD_RPCPORT=$[ 21111 + $NFS_INSTANCE ] + NFSD_RPCPORT=$[ 11111 + $NFS_INSTANCE ] + NFSD_PORT=$[ 3049 + 2 * $NFS_INSTANCE ] + MOUNTD_PORT=$[ 3048 + 2 * $NFS_INSTANCE ] + UNFS_OPTS="nfsvers=3,port=$NFSD_PORT,mountprog=$MOUNTD_RPCPORT,nfsprog=$NFSD_RPCPORT,udp,mountport=$MOUNTD_PORT" + + PSEUDO_LOCALSTATEDIR=~/.runqemu-sdk/pseudo + export PSEUDO_LOCALSTATEDIR + + # Start the userspace NFS server + echo "runqemu-export-rootfs restart $ROOTFS" + runqemu-export-rootfs restart $ROOTFS + if [ $? != 0 ]; then + return 1 + fi + NFSRUNNING="true" +fi + + +set_mem_size() { + if [ ! -z "$mem_set" ] ; then + #Get memory setting size from user input + mem_size=`echo $mem_set | sed 's/-m[[:space:]] *//'` + else + mem_size=$1 + fi + # QEMU_MEMORY has 'M' appended to mem_size + QEMU_MEMORY="$mem_size"M + +} + +config_qemuarm() { + set_mem_size 128 + QEMU=qemu-system-arm + MACHINE_SUBTYPE=versatilepb + export QEMU_AUDIO_DRV="none" + QEMU_UI_OPTIONS="$QEMU_UI_OPTIONS" + if [ "${FSTYPE:0:3}" = "ext" -o "$FSTYPE" = "btrfs" -o "$FSTYPE" = "wic" ]; then + KERNCMDLINE="root=$DROOT rw console=ttyAMA0,115200 console=tty $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY highres=off" + QEMUOPTIONS="$QEMU_NETWORK_CMD -M ${MACHINE_SUBTYPE} $ROOTFS_OPTIONS -no-reboot $QEMU_UI_OPTIONS" + fi + if [ "$FSTYPE" = "nfs" ]; then + if [ "$NFS_SERVER" = "192.168.7.1" -a ! -d "$NFS_DIR" ]; then + echo "Error: NFS mount point $ROOTFS doesn't exist" + cleanup + return 1 + fi + KERNCMDLINE="root=/dev/nfs nfsroot=$NFS_SERVER:$NFS_DIR,$UNFS_OPTS rw console=ttyAMA0,115200 $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_NETWORK_CMD -M ${MACHINE_SUBTYPE} --no-reboot $QEMU_UI_OPTIONS" + fi + if [ "$MACHINE" = "qemuarmv6" ]; then + QEMUOPTIONS="$QEMUOPTIONS -cpu arm1136" + fi + if [ "$MACHINE" = "qemuarmv7" ]; then + QEMUOPTIONS="$QEMUOPTIONS -cpu cortex-a8" + fi +} + +config_qemuarm64() { + set_mem_size 512 + QEMU=qemu-system-aarch64 + + QEMU_NETWORK_CMD="-netdev tap,id=net0,ifname=$TAP,script=no,downscript=no -device virtio-net-device,netdev=net0 " + DROOT="/dev/vda" + ROOTFS_OPTIONS="-drive id=disk0,file=$ROOTFS,if=none,format=raw -device virtio-blk-device,drive=disk0" + + export QEMU_AUDIO_DRV="none" + if [ "x$SERIALSTDIO" = "x" ] ; then + QEMU_UI_OPTIONS="-nographic" + else + QEMU_UI_OPTIONS="" + fi + if [ "${FSTYPE:0:3}" = "ext" -o "$FSTYPE" = "btrfs" -o "$FSTYPE" = "wic" ]; then + KERNCMDLINE="root=$DROOT rw console=ttyAMA0,38400 mem=$QEMU_MEMORY highres=off $KERNEL_NETWORK_CMD" + # qemu-system-aarch64 only support '-machine virt -cpu cortex-a57' for now + QEMUOPTIONS="$QEMU_NETWORK_CMD -machine virt -cpu cortex-a57 $ROOTFS_OPTIONS $QEMU_UI_OPTIONS" + fi + if [ "$FSTYPE" = "nfs" ]; then + if [ "$NFS_SERVER" = "192.168.7.1" -a ! -d "$NFS_DIR" ]; then + echo "Error: NFS mount point $ROOTFS doesn't exist" + cleanup + return 1 + fi + KERNCMDLINE="root=/dev/nfs nfsroot=$NFS_SERVER:$NFS_DIR,$UNFS_OPTS rw console=ttyAMA0,38400 mem=$QEMU_MEMORY highres=off $KERNEL_NETWORK_CMD" + QEMUOPTIONS="$QEMU_NETWORK_CMD -machine virt -cpu cortex-a57 $QEMU_UI_OPTIONS" + fi +} + +config_qemux86() { + set_mem_size 256 + QEMU=qemu-system-i386 + if [ "$KVM_ACTIVE" = "yes" ]; then + CPU_SUBTYPE=kvm32 + else + CPU_SUBTYPE=qemu32 + fi + if [ ! -z "$vga_option" ]; then + QEMU_UI_OPTIONS="$QEMU_UI_OPTIONS" + else + QEMU_UI_OPTIONS="$QEMU_UI_OPTIONS -vga vmware" + fi + if [ "${FSTYPE:0:3}" = "ext" -o "$FSTYPE" = "btrfs" -o "$FSTYPE" = "wic" ]; then + KERNCMDLINE="vga=0 uvesafb.mode_option=640x480-32 root=$DROOT rw mem=$QEMU_MEMORY $KERNEL_NETWORK_CMD" + QEMUOPTIONS="$QEMU_NETWORK_CMD -cpu $CPU_SUBTYPE $ROOTFS_OPTIONS $QEMU_UI_OPTIONS" + fi + if [ "${FSTYPE:0:4}" = "cpio" ]; then + KERNCMDLINE="vga=0 uvesafb.mode_option=640x480-32 root=/dev/ram0 rw mem=$QEMU_MEMORY $KERNEL_NETWORK_CMD" + QEMUOPTIONS="$QEMU_NETWORK_CMD -cpu $CPU_SUBTYPE -initrd $ROOTFS $QEMU_UI_OPTIONS" + fi + + if [ "$FSTYPE" = "nfs" ]; then + if [ "$NFS_SERVER" = "192.168.7.1" -a ! -d "$NFS_DIR" ]; then + echo "Error: NFS mount point $ROOTFS doesn't exist." + cleanup + return 1 + fi + KERNCMDLINE="root=/dev/nfs nfsroot=$NFS_SERVER:$NFS_DIR,$UNFS_OPTS rw $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_NETWORK_CMD $QEMU_UI_OPTIONS" + fi + if [ "$IS_VM" = "true" ]; then + QEMUOPTIONS="$QEMU_NETWORK_CMD $QEMU_UI_OPTIONS" + fi + # Currently oprofile's event based interrupt mode doesn't work(Bug #828) in + # qemux86 and qemux86-64. We can use timer interrupt mode for now. + KERNCMDLINE="$KERNCMDLINE oprofile.timer=1" +} + +config_qemux86_64() { + set_mem_size 256 + QEMU=qemu-system-x86_64 + if [ "$KVM_ACTIVE" = "yes" ]; then + CPU_SUBTYPE=kvm64 + else + CPU_SUBTYPE=core2duo + fi + if [ ! -z "$vga_option" ]; then + QEMU_UI_OPTIONS="$QEMU_UI_OPTIONS" + else + QEMU_UI_OPTIONS="$QEMU_UI_OPTIONS -vga vmware" + fi + if [ "${FSTYPE:0:3}" = "ext" -o "$FSTYPE" = "btrfs" -o "$FSTYPE" = "wic" ]; then + KERNCMDLINE="vga=0 uvesafb.mode_option=640x480-32 root=$DROOT rw mem=$QEMU_MEMORY $KERNEL_NETWORK_CMD" + QEMUOPTIONS="$QEMU_NETWORK_CMD -cpu $CPU_SUBTYPE $ROOTFS_OPTIONS $QEMU_UI_OPTIONS" + fi + if [ "$FSTYPE" = "nfs" ]; then + if [ "x$ROOTFS" = "x" ]; then + ROOTFS=/srv/nfs/qemux86-64 + fi + if [ ! -d "$ROOTFS" ]; then + echo "Error: NFS mount point $ROOTFS doesn't exist." + cleanup + return 1 + fi + KERNCMDLINE="root=/dev/nfs nfsroot=$NFS_SERVER:$NFS_DIR,$UNFS_OPTS rw $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_NETWORK_CMD -cpu $CPU_SUBTYPE $QEMU_UI_OPTIONS" + fi + if [ "$IS_VM" = "true" ]; then + QEMUOPTIONS="$QEMU_NETWORK_CMD -cpu $CPU_SUBTYPE $QEMU_UI_OPTIONS" + fi + # Currently oprofile's event based interrupt mode doesn't work(Bug #828) in + # qemux86 and qemux86-64. We can use timer interrupt mode for now. + KERNCMDLINE="$KERNCMDLINE oprofile.timer=1" +} + +config_qemumips() { + set_mem_size 256 + case "$MACHINE" in + qemumips) QEMU=qemu-system-mips ;; + qemumipsel) QEMU=qemu-system-mipsel ;; + qemumips64) QEMU=qemu-system-mips64 ;; + esac + MACHINE_SUBTYPE=malta + QEMU_UI_OPTIONS="-vga cirrus $QEMU_UI_OPTIONS" + if [ "${FSTYPE:0:3}" = "ext" -o "$FSTYPE" = "btrfs" -o "$FSTYPE" = "wic" ]; then + #KERNCMDLINE="root=/dev/hda console=ttyS0 console=tty0 $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + KERNCMDLINE="root=$DROOT rw console=ttyS0 console=tty $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_NETWORK_CMD -M $MACHINE_SUBTYPE $ROOTFS_OPTIONS -no-reboot $QEMU_UI_OPTIONS" + fi + if [ "$FSTYPE" = "nfs" ]; then + if [ "$NFS_SERVER" = "192.168.7.1" -a ! -d "$NFS_DIR" ]; then + echo "Error: NFS mount point $ROOTFS doesn't exist" + cleanup + return 1 + fi + KERNCMDLINE="root=/dev/nfs console=ttyS0 console=tty nfsroot=$NFS_SERVER:$NFS_DIR,$UNFS_OPTS rw $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_NETWORK_CMD -M $MACHINE_SUBTYPE -no-reboot $QEMU_UI_OPTIONS" + fi +} + +config_qemuppc() { + set_mem_size 256 + QEMU=qemu-system-ppc + MACHINE_SUBTYPE=mac99 + CPU_SUBTYPE=G4 + QEMU_UI_OPTIONS="$QEMU_UI_OPTIONS" + if [ "$SLIRP_ENABLED" = "yes" ]; then + QEMU_NETWORK_CMD="" + else + QEMU_NETWORK_CMD="-net nic,model=pcnet $QEMU_TAP_CMD" + fi + if [ "${FSTYPE:0:3}" = "ext" -o "$FSTYPE" = "btrfs" -o "$FSTYPE" = "wic" ]; then + KERNCMDLINE="root=$DROOT rw console=ttyS0 console=tty $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_NETWORK_CMD -cpu $CPU_SUBTYPE -M $MACHINE_SUBTYPE $ROOTFS_OPTIONS -no-reboot $QEMU_UI_OPTIONS" + fi + if [ "$FSTYPE" = "nfs" ]; then + if [ "$NFS_SERVER" = "192.168.7.1" -a ! -d "$NFS_DIR" ]; then + echo "Error: NFS mount point $ROOTFS doesn't exist" + cleanup + return 1 + fi + KERNCMDLINE="root=/dev/nfs console=ttyS0 console=tty nfsroot=$NFS_SERVER:$NFS_DIR,$UNFS_OPTS rw $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_NETWORK_CMD -cpu $CPU_SUBTYPE -M $MACHINE_SUBTYPE -no-reboot $QEMU_UI_OPTIONS" + fi +} + +config_qemush4() { + set_mem_size 1024 + QEMU=qemu-system-sh4 + MACHINE_SUBTYPE=r2d + QEMU_UI_OPTIONS="$QEMU_UI_OPTIONS" + if [ "${FSTYPE:0:3}" = "ext" -o "$FSTYPE" = "btrfs" -o "$FSTYPE" = "wic" ]; then + #KERNCMDLINE="root=/dev/hda console=ttyS0 console=tty0 $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + KERNCMDLINE="root=/dev/hda rw console=ttySC1 noiotrap earlyprintk=sh-sci.1 console=tty $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_NETWORK_CMD -M $MACHINE_SUBTYPE -hda $ROOTFS -no-reboot $QEMU_UI_OPTIONS -monitor null -serial vc -serial stdio" + SERIALSTDIO="1" + fi + if [ "$FSTYPE" = "nfs" ]; then + if [ "$NFS_SERVER" = "192.168.7.1" -a ! -d "$NFS_DIR" ]; then + echo "Error: NFS mount point $ROOTFS doesn't exist" + cleanup + return 1 + fi + KERNCMDLINE="root=/dev/nfs console=ttySC1 noiotrap earlyprintk=sh-sci.1 console=tty nfsroot=$NFS_SERVER:$NFS_DIR,$UNFS_OPTS rw $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_NETWORK_CMD -M $MACHINE_SUBTYPE -no-reboot $QEMU_UI_OPTIONS -monitor null -serial vc -serial stdio" + SERIALSTDIO="1" + fi +} + +config_qemuzynq() { + set_mem_size 1024 + QEMU=qemu-system-arm + QEMU_SYSTEM_OPTIONS="$QEMU_NETWORK_CMD -M xilinx-zynq-a9 -serial null -serial mon:stdio -dtb $KERNEL-$MACHINE.dtb" + # zynq serial ports are named 'ttyPS0' and 'ttyPS1', fixup the default values + SCRIPT_KERNEL_OPT=$(echo "$SCRIPT_KERNEL_OPT" | sed 's/console=ttyS/console=ttyPS/g') + if [ "${FSTYPE:0:3}" = "ext" -o "${FSTYPE:0:4}" = "cpio" ]; then + KERNCMDLINE="earlyprintk root=/dev/ram rw $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_SYSTEM_OPTIONS -initrd $ROOTFS" + fi +} + +config_qemumicroblaze() { + set_mem_size 256 + QEMU=qemu-system-microblazeel + QEMU_SYSTEM_OPTIONS="$QEMU_NETWORK_CMD -M petalogix-ml605 -serial mon:stdio" + if [ "${FSTYPE:0:3}" = "ext" -o "${FSTYPE:0:4}" = "cpio" ]; then + KERNCMDLINE="earlyprintk root=/dev/ram rw $KERNEL_NETWORK_CMD mem=$QEMU_MEMORY" + QEMUOPTIONS="$QEMU_SYSTEM_OPTIONS -initrd $ROOTFS" + fi +} + +case "$MACHINE" in + "qemuarm" | "qemuarmv6" | "qemuarmv7") + config_qemuarm + ;; + "qemuarm64") + config_qemuarm64 + ;; + "qemux86") + config_qemux86 + ;; + "qemux86-64") + config_qemux86_64 + ;; + "qemumips" | "qemumipsel" | "qemumips64") + config_qemumips + ;; + "qemuppc") + config_qemuppc + ;; + "qemush4") + config_qemush4 + ;; + "qemuzynq") + config_qemuzynq + ;; + "qemumicroblaze") + config_qemumicroblaze + ;; + *) + echo "Error: Unsupported machine type $MACHINE" + return 1 + ;; +esac + +# We need to specify -m <mem_size> to overcome a bug in qemu 0.14.0 +# https://bugs.launchpad.net/ubuntu/+source/qemu-kvm/+bug/584480 +if [ -z "$mem_set" ] ; then + SCRIPT_QEMU_EXTRA_OPT="$SCRIPT_QEMU_EXTRA_OPT -m $mem_size" +fi + +if [ "${FSTYPE:0:3}" = "ext" ]; then + KERNCMDLINE="$KERNCMDLINE rootfstype=$FSTYPE" +fi + +if [ "$FSTYPE" = "cpio.gz" ]; then + QEMUOPTIONS="-initrd $ROOTFS -nographic" + KERNCMDLINE="root=/dev/ram0 console=ttyS0 debugshell" +fi + +if [ "$FSTYPE" = "iso" ]; then + QEMUOPTIONS="$QEMU_NETWORK_CMD -cdrom $ROOTFS $QEMU_UI_OPTIONS" +fi + +if [ "x$QEMUOPTIONS" = "x" ]; then + echo "Error: Unable to support this combination of options" + cleanup + return 1 +fi + +if [ "$TCPSERIAL_PORTNUM" != "" ]; then + if [ "$MACHINE" = "qemuarm64" ]; then + SCRIPT_QEMU_EXTRA_OPT="$SCRIPT_QEMU_EXTRA_OPT -device virtio-serial-device -chardev socket,id=virtcon,port=$TCPSERIAL_PORTNUM,host=127.0.0.1 -device virtconsole,chardev=virtcon" + else + SCRIPT_QEMU_EXTRA_OPT="$SCRIPT_QEMU_EXTRA_OPT -serial tcp:127.0.0.1:$TCPSERIAL_PORTNUM" + fi +fi + +PATH=$OECORE_NATIVE_SYSROOT/usr/bin:$PATH + +QEMUBIN=`which $QEMU 2> /dev/null` +if [ ! -x "$QEMUBIN" ]; then + echo "Error: No QEMU binary '$QEMU' could be found." + cleanup + return 1 +fi + +NEED_GL=`ldd $QEMUBIN/$QEMU 2>&1 | grep libGLU` +# We can't run without a libGL.so +if [ "$NEED_GL" != "" ]; then + libgl='no' + + [ -e /usr/lib/libGL.so -a -e /usr/lib/libGLU.so ] && libgl='yes' + [ -e /usr/lib64/libGL.so -a -e /usr/lib64/libGLU.so ] && libgl='yes' + [ -e /usr/lib/*-linux-gnu/libGL.so -a -e /usr/lib/*-linux-gnu/libGLU.so ] && libgl='yes' + + if [ "$libgl" != 'yes' ]; then + echo "You need libGL.so and libGLU.so to exist in your library path to run the QEMU emulator. + Ubuntu package names are: libgl1-mesa-dev and libglu1-mesa-dev. + Fedora package names are: mesa-libGL-devel mesa-libGLU-devel." + return 1; + fi +fi + +do_quit() { + cleanup + return 1 +} + +trap do_quit INT TERM QUIT + +# qemu got segfault if linked with nVidia's libgl +GL_LD_PRELOAD=$LD_PRELOAD + +if ldd $QEMUBIN | grep -i nvidia &> /dev/null +then +cat << EOM +WARNING: nVidia proprietary OpenGL libraries detected. +nVidia's OpenGL libraries are known to have compatibility issues with qemu, +resulting in a segfault. Please uninstall these drivers or ensure the mesa libGL +libraries precede nvidia's via LD_PRELOAD(Already do it on Ubuntu 10). +EOM + +# Automatically use Ubuntu system's mesa libGL, other distro can add its own path +if grep -i ubuntu /etc/lsb-release &> /dev/null +then + # precede nvidia's driver on Ubuntu 10 + UBUNTU_MAIN_VERSION=`cat /etc/lsb-release |grep DISTRIB_RELEASE |cut -d= -f 2| cut -d. -f 1` + if [ "$UBUNTU_MAIN_VERSION" = "10" ]; + then + GL_PATH="" + if test -e /usr/lib/libGL.so + then + GL_PATH="/usr/lib/libGL.so" + elif test -e /usr/lib/x86_64-linux-gnu/libGL.so + then + GL_PATH="/usr/lib/x86_64-linux-gnu/libGL.so" + fi + + echo "Skip nVidia's libGL on Ubuntu 10!" + GL_LD_PRELOAD="$GL_PATH $LD_PRELOAD" + fi +fi +fi + +if [ "x$SERIALSTDIO" = "x1" ]; then + echo "Interrupt character is '^]'" + stty intr ^] +fi + + +# Preserve the multiplexing behavior for the monitor that would be there based +# on whether nographic is used. +if echo "$QEMUOPTIONS $SERIALOPTS $SCRIPT_QEMU_OPT $SCRIPT_QEMU_EXTRA_OPT" | grep -- "-nographic"; then + FIRST_SERIAL_OPT="-serial mon:stdio" +else + FIRST_SERIAL_OPT="-serial mon:vc" +fi + +# qemuarm64 uses virtio for any additional serial ports so the normal mechanism +# of using -serial will not work +if [ "$MACHINE" = "qemuarm64" ]; then + SECOND_SERIAL_OPT="$SCRIPT_QEMU_EXTRA_OPT -device virtio-serial-device -chardev null,id=virtcon -device virtconsole,chardev=virtcon" +else + SECOND_SERIAL_OPT="-serial null" +fi + +# We always want a ttyS1. Since qemu by default adds a serial port when +# nodefaults is not specified, it seems that all that would be needed is to +# make sure a "-serial" is there. However, it appears that when "-serial" is +# specified, it ignores the default serial port that is normally added. +# So here we make sure to add two -serial if there are none. And only one +# if there is one -serial already. +NUM_SERIAL_OPTS=`echo $QEMUOPTIONS $SERIALOPTS $SCRIPT_QEMU_OPT $SCRIPT_QEMU_EXTRA_OPT | sed -e 's/ /\n/g' | grep --count -- -serial` + +if [ "$NUM_SERIAL_OPTS" = "0" ]; then + SCRIPT_QEMU_EXTRA_OPT="$SCRIPT_QEMU_EXTRA_OPT $FIRST_SERIAL_OPT $SECOND_SERIAL_OPT" +elif [ "$NUM_SERIAL_OPTS" = "1" ]; then + SCRIPT_QEMU_EXTRA_OPT="$SCRIPT_QEMU_EXTRA_OPT $SECOND_SERIAL_OPT" +fi + +echo "Running $QEMU..." +# -no-reboot is a mandatory option - see bug #100 +if [ "$IS_VM" = "true" ]; then + # Check root=/dev/sdX or root=/dev/vdX + [ ! -e "$VM" ] && error "VM image is not found!" + if grep -q 'root=/dev/sd' $VM; then + echo "Using scsi drive" + VM_DRIVE="-drive if=none,id=hd,file=$VM -device virtio-scsi-pci,id=scsi -device scsi-hd,drive=hd" + elif grep -q 'root=/dev/hd' $VM; then + echo "Using ide drive" + VM_DRIVE="$VM" + else + echo "Using virtio block drive" + VM_DRIVE="-drive if=virtio,file=$VM" + fi + QEMU_FIRE="$QEMUBIN $VM_DRIVE $QEMUOPTIONS $SERIALOPTS -no-reboot $SCRIPT_QEMU_OPT $SCRIPT_QEMU_EXTRA_OPT" + echo $QEMU_FIRE + LD_PRELOAD="$GL_LD_PRELOAD" $QEMU_FIRE +elif [ "$FSTYPE" = "iso" -o "$FSTYPE" = "wic" ]; then + QEMU_FIRE="$QEMUBIN $QEMUOPTIONS $SERIALOPTS -no-reboot $SCRIPT_QEMU_OPT $SCRIPT_QEMU_EXTRA_OPT" + echo $QEMU_FIRE + LD_PRELOAD="$GL_LD_PRELOAD" $QEMU_FIRE +else + QEMU_FIRE="$QEMUBIN -kernel $KERNEL $QEMUOPTIONS $SLIRP_CMD $SERIALOPTS -no-reboot $SCRIPT_QEMU_OPT $SCRIPT_QEMU_EXTRA_OPT" + echo $QEMU_FIRE -append '"'$KERNCMDLINE $SCRIPT_KERNEL_OPT'"' + LD_PRELOAD="$GL_LD_PRELOAD" $QEMU_FIRE -append "$KERNCMDLINE $SCRIPT_KERNEL_OPT" +fi +ret=$? +if [ "$SLIRP_ENABLED" != "yes" ]; then + cleanup +fi + +#set the original stty values before exit +stty ${ORIG_STTY} +trap - INT TERM QUIT + +return $ret diff --git a/scripts/runqemu.README b/scripts/runqemu.README new file mode 100644 index 0000000..5908d83 --- /dev/null +++ b/scripts/runqemu.README @@ -0,0 +1,42 @@ +Using OE images with QEMU +========================= + +OE-Core can generate qemu bootable kernels and images with can be used +on a desktop system. The scripts currently support booting ARM, MIPS, PowerPC +and x86 (32 and 64 bit) images. The scripts can be used within the OE build +system or externaly. + +The runqemu script is run as: + + runqemu <machine> <zimage> <filesystem> + +where: + + <machine> is the machine/architecture to use (qemuarm/qemumips/qemuppc/qemux86/qemux86-64) + <zimage> is the path to a kernel (e.g. zimage-qemuarm.bin) + <filesystem> is the path to an ext2 image (e.g. filesystem-qemuarm.ext2) or an nfs directory + +If <machine> isn't specified, the script will try to detect the machine name +from the name of the <zimage> file. + +If <filesystem> isn't specified, nfs booting will be assumed. + +When used within the build system, it will default to qemuarm, ext2 and the last kernel and +core-image-sato-sdk image built by the build system. If an sdk image isn't present it will look +for sato and minimal images. + +Full usage instructions can be seen by running the command with no options specified. + + +Notes +===== + + - The scripts run qemu using sudo. Change perms on /dev/net/tun to + run as non root. The runqemu-gen-tapdevs script can also be used by + root to prepopulate the appropriate network devices. + - You can access the host computer at 192.168.7.1 within the image. + - Your qemu system will be accessible as 192.16.7.2. + - The script extracts the root filesystem specified under pseudo and sets up a userspace + NFS server to share the image over by default meaning the filesystem can be accessed by + both the host and guest systems. + diff --git a/scripts/send-error-report b/scripts/send-error-report new file mode 100755 index 0000000..a29feff --- /dev/null +++ b/scripts/send-error-report @@ -0,0 +1,200 @@ +#!/usr/bin/env python + +# Sends an error report (if the report-error class was enabled) to a +# remote server. +# +# Copyright (C) 2013 Intel Corporation +# Author: Andreea Proca <andreea.b.proca@intel.com> +# Author: Michael Wood <michael.g.wood@intel.com> + +import urllib2 +import sys +import json +import os +import subprocess +import argparse +import logging + +scripts_lib_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib') +sys.path.insert(0, scripts_lib_path) +import argparse_oe + +version = "0.3" + +log = logging.getLogger("send-error-report") +logging.basicConfig(format='%(levelname)s: %(message)s') + +def getPayloadLimit(url): + req = urllib2.Request(url, None) + try: + response = urllib2.urlopen(req) + except urllib2.URLError as e: + # Use this opportunity to bail out if we can't even contact the server + log.error("Could not contact server: " + url) + log.error(e.reason) + sys.exit(1) + try: + ret = json.loads(response.read()) + max_log_size = ret.get('max_log_size', 0) + return int(max_log_size) + except: + pass + + return 0 + +def ask_for_contactdetails(): + print("Please enter your name and your email (optionally), they'll be saved in the file you send.") + username = raw_input("Name (required): ") + email = raw_input("E-mail (not required): ") + return username, email + +def edit_content(json_file_path): + edit = raw_input("Review information before sending? (y/n): ") + if 'y' in edit or 'Y' in edit: + editor = os.environ.get('EDITOR', None) + if editor: + subprocess.check_call([editor, json_file_path]) + else: + log.error("Please set your EDITOR value") + sys.exit(1) + return True + return False + +def prepare_data(args): + # attempt to get the max_log_size from the server's settings + max_log_size = getPayloadLimit("http://"+args.server+"/ClientPost/JSON") + + if not os.path.isfile(args.error_file): + log.error("No data file found.") + sys.exit(1) + + home = os.path.expanduser("~") + userfile = os.path.join(home, ".oe-send-error") + + try: + with open(userfile, 'r') as userfile_fp: + if len(args.name) == 0: + args.name = userfile_fp.readline() + else: + #use empty readline to increment the fp + userfile_fp.readline() + + if len(args.email) == 0: + args.email = userfile_fp.readline() + except: + pass + + if args.assume_yes == True and len(args.name) == 0: + log.error("Name needs to be provided either via "+userfile+" or as an argument (-n).") + sys.exit(1) + + while len(args.name) <= 0 and len(args.name) < 50: + print("\nName needs to be given and must not more than 50 characters.") + args.name, args.email = ask_for_contactdetails() + + with open(userfile, 'w') as userfile_fp: + userfile_fp.write(args.name.strip() + "\n") + userfile_fp.write(args.email.strip() + "\n") + + with open(args.error_file, 'r') as json_fp: + data = json_fp.read() + + jsondata = json.loads(data) + jsondata['username'] = args.name.strip() + jsondata['email'] = args.email.strip() + jsondata['link_back'] = args.link_back.strip() + # If we got a max_log_size then use this to truncate to get the last + # max_log_size bytes from the end + if max_log_size != 0: + for fail in jsondata['failures']: + if len(fail['log']) > max_log_size: + print "Truncating log to allow for upload" + fail['log'] = fail['log'][-max_log_size:] + + data = json.dumps(jsondata, indent=4, sort_keys=True) + + # Write back the result which will contain all fields filled in and + # any post processing done on the log data + with open(args.error_file, "w") as json_fp: + if data: + json_fp.write(data) + + + if args.assume_yes == False and edit_content(args.error_file): + #We'll need to re-read the content if we edited it + with open(args.error_file, 'r') as json_fp: + data = json_fp.read() + + return data + + +def send_data(data, args): + headers={'Content-type': 'application/json', 'User-Agent': "send-error-report/"+version} + + if args.json: + url = "http://"+args.server+"/ClientPost/JSON/" + else: + url = "http://"+args.server+"/ClientPost/" + + req = urllib2.Request(url, data=data, headers=headers) + try: + response = urllib2.urlopen(req) + except urllib2.HTTPError, e: + logging.error(e.reason) + sys.exit(1) + + print response.read() + + +if __name__ == '__main__': + arg_parse = argparse_oe.ArgumentParser(description="This scripts will send an error report to your specified error-report-web server.") + + arg_parse.add_argument("error_file", + help="Generated error report file location", + type=str) + + arg_parse.add_argument("-y", + "--assume-yes", + help="Assume yes to all queries and do not prompt", + action="store_true") + + arg_parse.add_argument("-s", + "--server", + help="Server to send error report to", + type=str, + default="errors.yoctoproject.org") + + arg_parse.add_argument("-e", + "--email", + help="Email address to be used for contact", + type=str, + default="") + + arg_parse.add_argument("-n", + "--name", + help="Submitter name used to identify your error report", + type=str, + default="") + + arg_parse.add_argument("-l", + "--link-back", + help="A url to link back to this build from the error report server", + type=str, + default="") + + arg_parse.add_argument("-j", + "--json", + help="Return the result in json format, silences all other output", + action="store_true") + + + + args = arg_parse.parse_args() + + if (args.json == False): + print "Preparing to send errors to: "+args.server + + data = prepare_data(args) + send_data(data, args) + + sys.exit(0) diff --git a/scripts/send-pull-request b/scripts/send-pull-request new file mode 100755 index 0000000..575549d --- /dev/null +++ b/scripts/send-pull-request @@ -0,0 +1,179 @@ +#!/bin/bash +# +# Copyright (c) 2010-2011, Intel Corporation. +# All Rights Reserved +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See +# the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +# +# This script is intended to be used to send a patch series prepared by the +# create-pull-request script to Open Embedded and The Yocto Project, as well +# as to related projects and layers. +# + +AUTO=0 +AUTO_CL=0 +GITSOBCC="--suppress-cc=all" + +# Prevent environment leakage to these vars. +unset TO +unset CC +unset AUTO_CC +unset EXTRA_CC + +usage() +{ +cat <<EOM +Usage: $(basename $0) [-h] [-a] [-c] [[-t email]...] -p pull-dir + -a Send the cover letter to every recipient listed in Cc and + Signed-off-by lines found in the cover letter and the patches. + This option implies -c. + -c Expand the Cc list for the individual patches using the Cc and + Signed-off-by lines from the same patch. + -C Add extra CC to each email sent. + -p pull-dir Directory containing summary and patch files + -t email Explicitly add email to the recipients +EOM +} + +# Collect addresses from a patch into AUTO_CC +# $1: a patch file +harvest_recipients() +{ + PATCH=$1 + export IFS=$',\n' + for REGX in "^[Cc][Cc]: *" "^[Ss]igned-[Oo]ff-[Bb]y: *"; do + for EMAIL in $(sed '/^---$/q' $PATCH | grep -e "$REGX" | sed "s/$REGX//"); do + if [ "${AUTO_CC/$EMAIL/}" == "$AUTO_CC" ] && [ -n "$EMAIL" ]; then + if [ -z "$AUTO_CC" ]; then + AUTO_CC=$EMAIL; + else + AUTO_CC="$AUTO_CC,$EMAIL"; + fi + fi + done + done + unset IFS +} + +# Parse and verify arguments +while getopts "acC:hp:t:" OPT; do + case $OPT in + a) + AUTO=1 + GITSOBCC="--signed-off-by-cc" + AUTO_CL=1 + ;; + c) + AUTO=1 + GITSOBCC="--signed-off-by-cc" + ;; + C) + EXTRA_CC="$OPTARG" + ;; + h) + usage + exit 0 + ;; + p) + PDIR=${OPTARG%/} + if [ ! -d $PDIR ]; then + echo "ERROR: pull-dir \"$PDIR\" does not exist." + usage + exit 1 + fi + ;; + t) + if [ -n "$TO" ]; then + TO="$TO,$OPTARG" + else + TO="$OPTARG" + fi + ;; + esac +done + +if [ -z "$PDIR" ]; then + echo "ERROR: you must specify a pull-dir." + usage + exit 1 +fi + + +# Verify the cover letter is complete and free of tokens +if [ -e $PDIR/0000-cover-letter.patch ]; then + CL="$PDIR/0000-cover-letter.patch" + for TOKEN in SUBJECT BLURB; do + grep -q "*** $TOKEN HERE ***" "$CL" + if [ $? -eq 0 ]; then + echo "ERROR: Please edit $CL and try again (Look for '*** $TOKEN HERE ***')." + exit 1 + fi + done +else + echo "WARNING: No cover letter will be sent." +fi + +# Harvest emails from the generated patches and populate AUTO_CC. +if [ $AUTO_CL -eq 1 ]; then + for PATCH in $PDIR/*.patch; do + harvest_recipients $PATCH + done +fi + +AUTO_TO="$(git config sendemail.to)" +if [ -n "$AUTO_TO" ]; then + if [ -n "$TO" ]; then + TO="$TO,$AUTO_TO" + else + TO="$AUTO_TO" + fi +fi + +if [ -z "$TO" ] && [ -z "$AUTO_CC" ]; then + echo "ERROR: you have not specified any recipients." + usage + exit 1 +fi + + +# Convert the collected addresses into git-send-email argument strings +export IFS=$',' +GIT_TO=$(for R in $TO; do echo -n "--to='$R' "; done) +GIT_CC=$(for R in $AUTO_CC; do echo -n "--cc='$R' "; done) +GIT_EXTRA_CC=$(for R in $EXTRA_CC; do echo -n "--cc='$R' "; done) +unset IFS + +# Handoff to git-send-email. It will perform the send confirmation. +PATCHES=$(echo $PDIR/*.patch) +if [ $AUTO_CL -eq 1 ]; then + # Send the cover letter to every recipient, both specified as well as + # harvested. Then remove it from the patches list. + eval "git send-email $GIT_TO $GIT_CC $GIT_EXTRA_CC --confirm=always --no-chain-reply-to --suppress-cc=all $CL" + if [ $? -eq 1 ]; then + echo "ERROR: failed to send cover-letter with automatic recipients." + exit 1 + fi + PATCHES=${PATCHES/"$CL"/} +fi + +# Send the patch to the specified recipients and, if -c was specified, those git +# finds in this specific patch. +eval "git send-email $GIT_TO $GIT_EXTRA_CC --confirm=always --no-chain-reply-to $GITSOBCC $PATCHES" +if [ $? -eq 1 ]; then + echo "ERROR: failed to send patches." + exit 1 +fi diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh new file mode 100755 index 0000000..2ab450a --- /dev/null +++ b/scripts/sstate-cache-management.sh @@ -0,0 +1,469 @@ +#!/bin/bash + +# Copyright (c) 2012 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +# Global vars +cache_dir= +confirm= +fsym= +total_deleted=0 +verbose= +debug=0 + +usage () { + cat << EOF +Welcome to sstate cache management utilities. +sstate-cache-management.sh <OPTION> + +Options: + -h, --help + Display this help and exit. + + --cache-dir=<sstate cache dir> + Specify sstate cache directory, will use the environment + variable SSTATE_CACHE_DIR if it is not specified. + + --extra-archs=<arch1>,<arch2>...<archn> + Specify list of architectures which should be tested, this list + will be extended with native arch, allarch and empty arch. The + script won't be trying to generate list of available archs from + AVAILTUNES in tune files. + + --extra-layer=<layer1>,<layer2>...<layern> + Specify the layer which will be used for searching the archs, + it will search the meta and meta-* layers in the top dir by + default, and will search meta, meta-*, <layer1>, <layer2>, + ...<layern> when specified. Use "," as the separator. + + This is useless for --stamps-dir or when --extra-archs is used. + + -d, --remove-duplicated + Remove the duplicated sstate cache files of one package, only + the newest one will be kept. The duplicated sstate cache files + of one package must have the same arch, which means sstate cache + files with multiple archs are not considered duplicate. + + Conflicts with --stamps-dir. + + --stamps-dir=<dir1>,<dir2>...<dirn> + Specify the build directory's stamps directories, the sstate + cache file which IS USED by these build diretories will be KEPT, + other sstate cache files in cache-dir will be removed. Use "," + as the separator. For example: + --stamps-dir=build1/tmp/stamps,build2/tmp/stamps + + Conflicts with --remove-duplicated. + + -L, --follow-symlink + Remove both the symbol link and the destination file, default: no. + + -y, --yes + Automatic yes to prompts; assume "yes" as answer to all prompts + and run non-interactively. + + -v, --verbose + Explain what is being done. + + -D, --debug + Show debug info, repeat for more debug info. + +EOF +} + +if [ $# -lt 1 ]; then + usage + exit 0 +fi + +# Echo no files to remove +no_files () { + echo No files to remove +} + +# Echo nothing to do +do_nothing () { + echo Nothing to do +} + +# Read the input "y" +read_confirm () { + echo "$total_deleted out of $total_files files will be removed! " + if [ "$confirm" != "y" ]; then + echo "Do you want to continue (y/n)? " + while read confirm; do + [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \ + -o "$confirm" = "N" ] && break + echo "Invalid input \"$confirm\", please input 'y' or 'n': " + done + else + echo + fi +} + +# Print error information and exit. +echo_error () { + echo "ERROR: $1" >&2 + exit 1 +} + +# Generate the remove list: +# +# * Add .done/.siginfo to the remove list +# * Add destination of symlink to the remove list +# +# $1: output file, others: sstate cache file (.tgz) +gen_rmlist (){ + local rmlist_file="$1" + shift + local files="$@" + for i in $files; do + echo $i >> $rmlist_file + # Add the ".siginfo" + if [ -e $i.siginfo ]; then + echo $i.siginfo >> $rmlist_file + fi + # Add the destination of symlink + if [ -L "$i" ]; then + if [ "$fsym" = "y" ]; then + dest="`readlink -e $i`" + if [ -n "$dest" ]; then + echo $dest >> $rmlist_file + # Remove the .siginfo when .tgz is removed + if [ -f "$dest.siginfo" ]; then + echo $dest.siginfo >> $rmlist_file + fi + fi + fi + # Add the ".tgz.done" and ".siginfo.done" (may exist in the future) + base_fn="${i##/*/}" + t_fn="$base_fn.done" + s_fn="$base_fn.siginfo.done" + for d in $t_fn $s_fn; do + if [ -f $cache_dir/$d ]; then + echo $cache_dir/$d >> $rmlist_file + fi + done + fi + done +} + +# Remove the duplicated cache files for the pkg, keep the newest one +remove_duplicated () { + + local topdir + local oe_core_dir + local tunedirs + local all_archs + local all_machines + local ava_archs + local arch + local file_names + local sstate_files_list + local fn_tmp + local list_suffix=`mktemp` || exit 1 + + if [ -z "$extra_archs" ] ; then + # Find out the archs in all the layers + echo "Figuring out the archs in the layers ... " + oe_core_dir=$(dirname $(dirname $(readlink -e $0))) + topdir=$(dirname $oe_core_dir) + tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`" + [ -n "$tunedirs" ] || echo_error "Can't find the tune directory" + all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`" + all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'` + fi + + # Use the "_" to substitute "-", e.g., x86-64 to x86_64, but not for extra_archs which can be something like cortexa9t2-vfp-neon + # Sort to remove the duplicated ones + # Add allarch and builder arch (native) + builder_arch=$(uname -m) + all_archs="$(echo allarch $all_archs $all_machines $builder_arch \ + | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs" + echo "Done" + + # Total number of files including sstate-, .siginfo and .done files + total_files=`find $cache_dir -name 'sstate*' | wc -l` + # Save all the sstate files in a file + sstate_files_list=`mktemp` || exit 1 + find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz*' >$sstate_files_list + + echo "Figuring out the suffixes in the sstate cache dir ... " + sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $sstate_files_list | sort -u`" + echo "Done" + echo "The following suffixes have been found in the cache dir:" + echo $sstate_suffixes + + echo "Figuring out the archs in the sstate cache dir ... " + # Using this SSTATE_PKGSPEC definition it's 6th colon separated field + # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" + for arch in $all_archs; do + grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list + [ $? -eq 0 ] && ava_archs="$ava_archs $arch" + # ${builder_arch}_$arch used by toolchain sstate + grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list + [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch" + done + echo "Done" + echo "The following archs have been found in the cache dir:" + echo $ava_archs + echo "" + + # Save the file list which needs to be removed + local remove_listdir=`mktemp -d` || exit 1 + for suffix in $sstate_suffixes; do + if [ "$suffix" = "populate_lic" ] ; then + echo "Skipping populate_lic, because removing duplicates doesn't work correctly for them (use --stamps-dir instead)" + continue + fi + # Total number of files including .siginfo and .done files + total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list | wc -l 2>/dev/null` + total_tgz_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz$" $sstate_files_list | wc -l 2>/dev/null` + # Save the file list to a file, some suffix's file may not exist + grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list >$list_suffix 2>/dev/null + local deleted_tgz=0 + local deleted_files=0 + for ext in tgz tgz.siginfo tgz.done; do + echo "Figuring out the sstate:xxx_$suffix.$ext ... " + # Uniq BPNs + file_names=`for arch in $ava_archs ""; do + sed -ne "s%.*/sstate:\([^:]*\):[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$%\1%p" $list_suffix + done | sort -u` + + fn_tmp=`mktemp` || exit 1 + rm_list="$remove_listdir/sstate:xxx_$suffix" + for fn in $file_names; do + [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.${ext}" + for arch in $ava_archs ""; do + grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$" $list_suffix >$fn_tmp + if [ -s $fn_tmp ] ; then + [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix.${ext}:" && cat $fn_tmp + # Use the modification time + to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p') + [ $debug -gt 2 ] && echo "Considering to delete: $to_del" + # The sstate file which is downloaded from the SSTATE_MIRROR is + # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to + # it, so filter it out from the remove list if it should not be + # removed. + to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p') + [ $debug -gt 2 ] && echo "Considering to keep: $to_keep" + for k in $to_keep; do + if [ -L "$k" ]; then + # The symlink's destination + k_dest="`readlink -e $k`" + # Maybe it is the one in cache_dir + k_maybe="$cache_dir/${k##/*/}" + # Remove it from the remove list if they are the same. + if [ "$k_dest" = "$k_maybe" ]; then + to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`" + fi + fi + done + rm -f $fn_tmp + [ $debug -gt 2 ] && echo "Decided to delete: $to_del" + gen_rmlist $rm_list.$ext "$to_del" + fi + done + done + done + deleted_tgz=`cat $rm_list.* 2>/dev/null | grep ".tgz$" | wc -l` + deleted_files=`cat $rm_list.* 2>/dev/null | wc -l` + [ "$deleted_files" -gt 0 -a $debug -gt 0 ] && cat $rm_list.* + echo "($deleted_tgz out of $total_tgz_suffix .tgz files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)" + let total_deleted=$total_deleted+$deleted_files + done + deleted_tgz=0 + rm_old_list=$remove_listdir/sstate-old-filenames + find $cache_dir -name 'sstate-*.tgz' >$rm_old_list + [ -s "$rm_old_list" ] && deleted_tgz=`cat $rm_old_list | grep ".tgz$" | wc -l` + [ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l` + [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list + echo "($deleted_tgz .tgz files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)" + let total_deleted=$total_deleted+$deleted_files + + rm -f $list_suffix + rm -f $sstate_files_list + if [ $total_deleted -gt 0 ]; then + read_confirm + if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then + for list in `ls $remove_listdir/`; do + echo "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... " + # Remove them one by one to avoid the argument list too long error + for i in `cat $remove_listdir/$list`; do + rm -f $verbose $i + done + echo "Done" + done + echo "$total_deleted files have been removed!" + else + do_nothing + fi + else + no_files + fi + [ -d $remove_listdir ] && rm -fr $remove_listdir +} + +# Remove the sstate file by stamps dir, the file not used by the stamps dir +# will be removed. +rm_by_stamps (){ + + local cache_list=`mktemp` || exit 1 + local keep_list=`mktemp` || exit 1 + local rm_list=`mktemp` || exit 1 + local sums + local all_sums + + # Total number of files including sstate-, .siginfo and .done files + total_files=`find $cache_dir -type f -name 'sstate*' | wc -l` + # Save all the state file list to a file + find $cache_dir -type f -name 'sstate*' | sort -u -o $cache_list + + echo "Figuring out the suffixes in the sstate cache dir ... " + local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $cache_list | sort -u`" + echo "Done" + echo "The following suffixes have been found in the cache dir:" + echo $sstate_suffixes + + # Figure out all the md5sums in the stamps dir. + echo "Figuring out all the md5sums in stamps dir ... " + for i in $sstate_suffixes; do + # There is no "\.sigdata" but "_setcene" when it is mirrored + # from the SSTATE_MIRRORS, use them to figure out the sum. + sums=`find $stamps -maxdepth 3 -name "*.do_$i.*" \ + -o -name "*.do_${i}_setscene.*" | \ + sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \ + sed -e 's#\..*##' | sort -u` + all_sums="$all_sums $sums" + done + echo "Done" + + echo "Figuring out the files which will be removed ... " + for i in $all_sums; do + grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:${i}_.*" $cache_list >>$keep_list + done + echo "Done" + + if [ -s $keep_list ]; then + sort -u $keep_list -o $keep_list + to_del=`comm -1 -3 $keep_list $cache_list` + gen_rmlist $rm_list "$to_del" + let total_deleted=`cat $rm_list | sort -u | wc -w` + if [ $total_deleted -gt 0 ]; then + [ $debug -gt 0 ] && cat $rm_list | sort -u + read_confirm + if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then + echo "Removing sstate cache files ... ($total_deleted files)" + # Remove them one by one to avoid the argument list too long error + for i in `cat $rm_list | sort -u`; do + rm -f $verbose $i + done + echo "$total_deleted files have been removed" + else + do_nothing + fi + else + no_files + fi + else + echo_error "All files in cache dir will be removed! Abort!" + fi + + rm -f $cache_list + rm -f $keep_list + rm -f $rm_list +} + +# Parse arguments +while [ -n "$1" ]; do + case $1 in + --cache-dir=*) + cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e` + [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir" + shift + ;; + --remove-duplicated|-d) + rm_duplicated="y" + shift + ;; + --yes|-y) + confirm="y" + shift + ;; + --follow-symlink|-L) + fsym="y" + shift + ;; + --extra-archs=*) + extra_archs=`echo $1 | sed -e 's#^--extra-archs=##' -e 's#,# #g'` + [ -n "$extra_archs" ] || echo_error "Invalid extra arch parameter" + shift + ;; + --extra-layer=*) + extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'` + [ -n "$extra_layers" ] || echo_error "Invalid extra layer parameter" + for i in $extra_layers; do + l=`readlink -e $i` + if [ -d "$l" ]; then + layers="$layers $l" + else + echo_error "Can't find layer $i" + fi + done + shift + ;; + --stamps-dir=*) + stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'` + [ -n "$stamps" ] || echo_error "Invalid stamps dir $i" + for i in $stamps; do + [ -d "$i" ] || echo_error "Invalid stamps dir $i" + done + shift + ;; + --verbose|-v) + verbose="-v" + shift + ;; + --debug|-D) + debug=`expr $debug + 1` + echo "Debug level $debug" + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Invalid arguments $*" + echo_error "Try 'sstate-cache-management.sh -h' for more information." + ;; + esac +done + +# sstate cache directory, use environment variable SSTATE_CACHE_DIR +# if it was not specified, otherwise, error. +[ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR +[ -n "$cache_dir" ] || echo_error "No cache dir found!" +[ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\"" + +[ -n "$rm_duplicated" -a -n "$stamps" ] && \ + echo_error "Can not use both --remove-duplicated and --stamps-dir" + +[ "$rm_duplicated" = "y" ] && remove_duplicated +[ -n "$stamps" ] && rm_by_stamps +[ -z "$rm_duplicated" -a -z "$stamps" ] && \ + echo "What do you want to do?" +exit 0 diff --git a/scripts/sstate-diff-machines.sh b/scripts/sstate-diff-machines.sh new file mode 100755 index 0000000..056aa0a --- /dev/null +++ b/scripts/sstate-diff-machines.sh @@ -0,0 +1,172 @@ +#!/bin/bash + +# Used to compare sstate checksums between MACHINES. +# Execute script and compare generated list.M files. +# Using bash to have PIPESTATUS variable. + +# It's also usefull to keep older sstate checksums +# to be able to find out why something is rebuilding +# after updating metadata + +# $ diff \ +# sstate-diff/1349348392/fake-cortexa8/list.M \ +# sstate-diff/1349348392/fake-cortexa9/list.M \ +# | wc -l +# 538 + +# Then to compare sigdata use something like: +# $ ls sstate-diff/1349348392/*/armv7a-vfp-neon*/linux-libc-headers/*do_configure*sigdata* +# sstate-diff/1349348392/fake-cortexa8/armv7a-vfp-neon-oe-linux-gnueabi/linux-libc-headers/3.4.3-r0.do_configure.sigdata.cb73b3630a7b8191e72fc469c5137025 +# sstate-diff/1349348392/fake-cortexa9/armv7a-vfp-neon-oe-linux-gnueabi/linux-libc-headers/3.4.3-r0.do_configure.sigdata.f37ada177bf99ce8af85914df22b5a0b +# $ bitbake-diffsigs stamps.1349348392/*/armv7a-vfp-neon*/linux-libc-headers/*do_configure*sigdata* +# basehash changed from 8d0bd67bb1da6f68717760fc3ef43171 to e869fa61426e88e9c30726ba88a1216a +# Variable TUNE_CCARGS value changed from -march=armv7-a -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8 to -march=armv7-a -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a9 + +# Global vars +tmpdir= +machines= +targets= +default_machines="qemuarm qemux86 qemux86-64" +default_targets="core-image-base" +analyze="N" + +usage () { + cat << EOF +Welcome to utility to compare sstate checksums between different MACHINEs. +$0 <OPTION> + +Options: + -h, --help + Display this help and exit. + + --tmpdir=<tmpdir> + Specify tmpdir, will use the environment variable TMPDIR if it is not specified. + Something like /OE/oe-core/tmp-eglibc (no / at the end). + + --machines=<machines> + List of MACHINEs separated by space, will use the environment variable MACHINES if it is not specified. + Default value is "qemuarm qemux86 qemux86-64". + + --targets=<targets> + List of targets separated by space, will use the environment variable TARGETS if it is not specified. + Default value is "core-image-base". + + --analyze + Show the differences between MACHINEs. It assumes: + * First 2 MACHINEs in --machines parameter have the same TUNE_PKGARCH + * Third optional MACHINE has different TUNE_PKGARCH - only native and allarch recipes are compared). + * Next MACHINEs are ignored +EOF +} + +# Print error information and exit. +echo_error () { + echo "ERROR: $1" >&2 + exit 1 +} + +while [ -n "$1" ]; do + case $1 in + --tmpdir=*) + tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e` + [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir" + shift + ;; + --machines=*) + machines=`echo $1 | sed -e 's#^--machines="*\([^"]*\)"*#\1#'` + shift + ;; + --targets=*) + targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'` + shift + ;; + --analyze) + analyze="Y" + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Invalid arguments $*" + echo_error "Try '$0 -h' for more information." + ;; + esac +done + +# tmpdir directory, use environment variable TMPDIR +# if it was not specified, otherwise, error. +[ -n "$tmpdir" ] || tmpdir=$TMPDIR +[ -n "$tmpdir" ] || echo_error "No tmpdir found!" +[ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\"" +[ -n "$machines" ] || machines=$MACHINES +[ -n "$machines" ] || machines=$default_machines +[ -n "$targets" ] || targets=$TARGETS +[ -n "$targets" ] || targets=$default_targets + +OUTPUT=${tmpdir}/sstate-diff/`date "+%s"` +declare -i RESULT=0 + +for M in ${machines}; do + [ -d ${tmpdir}/stamps/ ] && find ${tmpdir}/stamps/ -name \*sigdata\* | xargs rm -f + mkdir -p ${OUTPUT}/${M} + export MACHINE=${M} + bitbake -S none ${targets} 2>&1 | tee -a ${OUTPUT}/${M}/log; + RESULT+=${PIPESTATUS[0]} + if ls ${tmpdir}/stamps/* >/dev/null 2>/dev/null ; then + cp -ra ${tmpdir}/stamps/* ${OUTPUT}/${M} + find ${OUTPUT}/${M} -name \*sigdata\* | sed "s#${OUTPUT}/${M}/##g" | sort > ${OUTPUT}/${M}/list + M_UNDERSCORE=`echo ${M} | sed 's/-/_/g'` + sed "s/${M_UNDERSCORE}/MACHINE/g; s/${M}/MACHINE/g" ${OUTPUT}/${M}/list | sort > ${OUTPUT}/${M}/list.M + find ${tmpdir}/stamps/ -name \*sigdata\* | xargs rm -f + else + printf "ERROR: no sigdata files were generated for MACHINE $M in ${tmpdir}/stamps\n"; + fi +done + +function compareSignatures() { + MACHINE1=$1 + MACHINE2=$2 + PATTERN="$3" + PRE_PATTERN="" + [ -n "${PATTERN}" ] || PRE_PATTERN="-v" + [ -n "${PATTERN}" ] || PATTERN="MACHINE" + for TASK in do_configure.sigdata do_populate_sysroot.sigdata do_package_write_ipk.sigdata; do + printf "\n\n === Comparing signatures for task ${TASK} between ${MACHINE1} and ${MACHINE2} ===\n" | tee -a ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log + diff ${OUTPUT}/${MACHINE1}/list.M ${OUTPUT}/${MACHINE2}/list.M | grep ${PRE_PATTERN} "${PATTERN}" | grep ${TASK} > ${OUTPUT}/signatures.${MACHINE2}.${TASK} + for i in `cat ${OUTPUT}/signatures.${MACHINE2}.${TASK} | sed 's#[^/]*/\([^/]*\)/.*#\1#g' | sort -u | xargs`; do + [ -e ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ] || echo "INFO: ${i} task ${TASK} doesn't exist in ${MACHINE1}" >&2 + [ -e ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ] || continue + [ -e ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}* ] || echo "INFO: ${i} task ${TASK} doesn't exist in ${MACHINE2}" >&2 + [ -e ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}* ] || continue + printf "ERROR: $i different signature for task ${TASK} between ${MACHINE1} and ${MACHINE2}\n"; + bitbake-diffsigs ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}*; + echo "$i" >> ${OUTPUT}/failed-recipes.log + echo + done | tee -a ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log + # don't create empty files + ERRORS=`grep "^ERROR.*" ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log | wc -l` + if [ "${ERRORS}" != "0" ] ; then + echo "ERROR: ${ERRORS} errors found in ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log" + RESULT+=${ERRORS} + fi + done +} + +function compareMachines() { + [ "$#" -ge 2 ] && compareSignatures $1 $2 + [ "$#" -ge 3 ] && compareSignatures $1 $3 "\(^< all\)\|\(^< x86_64-linux\)\|\(^< i586-linux\)" +} + +if [ "${analyze}" = "Y" ] ; then + compareMachines ${machines} +fi + +if [ "${RESULT}" != "0" -a -f ${OUTPUT}/failed-recipes.log ] ; then + cat ${OUTPUT}/failed-recipes.log | sort -u >${OUTPUT}/failed-recipes.log.u && mv ${OUTPUT}/failed-recipes.log.u ${OUTPUT}/failed-recipes.log + echo "ERROR: ${RESULT} issues were found in these recipes: `cat ${OUTPUT}/failed-recipes.log | xargs`" +fi + +echo "INFO: Output written in: ${OUTPUT}" +exit ${RESULT} diff --git a/scripts/sstate-sysroot-cruft.sh b/scripts/sstate-sysroot-cruft.sh new file mode 100755 index 0000000..b7ed8ea --- /dev/null +++ b/scripts/sstate-sysroot-cruft.sh @@ -0,0 +1,185 @@ +#!/bin/sh + +# Used to find files installed in sysroot which are not tracked by sstate manifest + +# Global vars +tmpdir= + +usage () { + cat << EOF +Welcome to sysroot cruft finding utility. +$0 <OPTION> + +Options: + -h, --help + Display this help and exit. + + --tmpdir=<tmpdir> + Specify tmpdir, will use the environment variable TMPDIR if it is not specified. + Something like /OE/oe-core/tmp-eglibc (no / at the end). + + --whitelist=<whitelist-file> + Text file, each line is regular expression for paths we want to ignore in resulting diff. + You can use diff file from the script output, if it contains only expected exceptions. + '#' is used as regexp delimiter, so you don't need to prefix forward slashes in paths. + ^ and $ is automatically added, so provide only the middle part. + Lines starting with '#' are ignored as comments. + All paths are relative to "sysroots" directory. + Directories don't end with forward slash. +EOF +} + +# Print error information and exit. +echo_error () { + echo "ERROR: $1" >&2 + exit 1 +} + +while [ -n "$1" ]; do + case $1 in + --tmpdir=*) + tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e` + [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir" + shift + ;; + --whitelist=*) + fwhitelist=`echo $1 | sed -e 's#^--whitelist=##' | xargs readlink -e` + [ -f "$fwhitelist" ] || echo_error "Invalid argument to --whitelist" + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Invalid arguments $*" + echo_error "Try '$0 -h' for more information." + ;; + esac +done + +# sstate cache directory, use environment variable TMPDIR +# if it was not specified, otherwise, error. +[ -n "$tmpdir" ] || tmpdir=$TMPDIR +[ -n "$tmpdir" ] || echo_error "No tmpdir found!" +[ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\"" + +OUTPUT=${tmpdir}/sysroot.cruft.`date "+%s"` + +# top level directories +WHITELIST="[^/]*" + +# generated by base-passwd recipe +WHITELIST="${WHITELIST} \ + .*/etc/group-\? \ + .*/etc/passwd-\? \ +" +# generated by pseudo-native +WHITELIST="${WHITELIST} \ + .*/var/pseudo \ + .*/var/pseudo/[^/]* \ +" + +# generated by package.bbclass:SHLIBSDIRS = "${PKGDATA_DIR}/${MLPREFIX}shlibs" +WHITELIST="${WHITELIST} \ + .*/shlibs \ + .*/pkgdata \ +" + +# generated by python +WHITELIST="${WHITELIST} \ + .*\.pyc \ + .*\.pyo \ + .*/__pycache__ \ +" + +# generated by lua +WHITELIST="${WHITELIST} \ + .*\.luac \ +" + +# generated by sgml-common-native +WHITELIST="${WHITELIST} \ + .*/etc/sgml/sgml-docbook.bak \ +" + +# generated by php +WHITELIST="${WHITELIST} \ + .*/usr/lib/php5/php/.channels/.* \ + .*/usr/lib/php5/php/.registry/.* \ + .*/usr/lib/php5/php/.depdb \ + .*/usr/lib/php5/php/.depdblock \ + .*/usr/lib/php5/php/.filemap \ + .*/usr/lib/php5/php/.lock \ +" + +# generated by toolchain +WHITELIST="${WHITELIST} \ + [^/]*-tcbootstrap/lib \ +" + +# generated by useradd.bbclass +WHITELIST="${WHITELIST} \ + [^/]*/home \ + [^/]*/home/xuser \ + [^/]*/home/xuser/.bashrc \ + [^/]*/home/xuser/.profile \ + [^/]*/home/builder \ + [^/]*/home/builder/.bashrc \ + [^/]*/home/builder/.profile \ +" + +# generated by image.py for WIC +# introduced in oe-core commit 861ce6c5d4836df1a783be3b01d2de56117c9863 +WHITELIST="${WHITELIST} \ + [^/]*/imgdata \ + [^/]*/imgdata/[^/]*\.env \ +" + +# generated by fontcache.bbclass +WHITELIST="${WHITELIST} \ + .*/var/cache/fontconfig/ \ +" + +SYSROOTS="`readlink -f ${tmpdir}`/sysroots/" + +mkdir ${OUTPUT} +find ${tmpdir}/sstate-control -name \*.populate-sysroot\* -o -name \*.populate_sysroot\* -o -name \*.package\* | xargs cat | grep sysroots | \ + sed 's#/$##g; s#///*#/#g' | \ + # work around for paths ending with / for directories and multiplied // (e.g. paths to native sysroot) + sort | sed "s#^${SYSROOTS}##g" > ${OUTPUT}/master.list.all.txt +sort -u ${OUTPUT}/master.list.all.txt > ${OUTPUT}/master.list.txt # -u because some directories are listed for more recipes +find ${tmpdir}/sysroots/ | \ + sort | sed "s#^${SYSROOTS}##g" > ${OUTPUT}/sysroot.list.txt + +diff ${OUTPUT}/master.list.all.txt ${OUTPUT}/master.list.txt > ${OUTPUT}/duplicates.txt +diff ${OUTPUT}/master.list.txt ${OUTPUT}/sysroot.list.txt > ${OUTPUT}/diff.all.txt + +grep "^> ." ${OUTPUT}/diff.all.txt | sed 's/^> //g' > ${OUTPUT}/diff.txt +for item in ${WHITELIST}; do + sed -i "\\#^${item}\$#d" ${OUTPUT}/diff.txt; + echo "${item}" >> ${OUTPUT}/used.whitelist.txt +done + +if [ -s "$fwhitelist" ] ; then + cat $fwhitelist >> ${OUTPUT}/used.whitelist.txt + cat $fwhitelist | grep -v '^#' | while read item; do + sed -i "\\#^${item}\$#d" ${OUTPUT}/diff.txt; + done +fi +# too many false positives for directories +# echo "Following files are installed in sysroot at least twice" +# cat ${OUTPUT}/duplicates + +RESULT=`cat ${OUTPUT}/diff.txt | wc -l` + +if [ "${RESULT}" != "0" ] ; then + echo "ERROR: ${RESULT} issues were found." + echo "ERROR: Following files are installed in sysroot, but not tracked by sstate:" + cat ${OUTPUT}/diff.txt +else + echo "INFO: All files are tracked by sstate or were explicitly ignored by this script" +fi + +echo "INFO: Output written in: ${OUTPUT}" +exit ${RESULT} diff --git a/scripts/swabber-strace-attach b/scripts/swabber-strace-attach new file mode 100755 index 0000000..bb0391a --- /dev/null +++ b/scripts/swabber-strace-attach @@ -0,0 +1,31 @@ +#!/usr/bin/env python +import os +import sys +import subprocess + +# Detach from the controlling terminal and parent process by forking twice to daemonize ourselves, +# then run the command passed as argv[1]. Send log data to argv[2]. + +pid = os.fork() +if (pid == 0): + os.setsid() + pid = os.fork() + if (pid != 0): + os._exit(0) +else: + sys.exit() + + +si = file(os.devnull, 'r') +so = file(sys.argv[2], 'w') +se = so + +# Replace those fds with our own +os.dup2(si.fileno(), sys.stdin.fileno()) +os.dup2(so.fileno(), sys.stdout.fileno()) +os.dup2(se.fileno(), sys.stderr.fileno()) + +ret = subprocess.call(sys.argv[1], shell=True) + +os._exit(ret) + diff --git a/scripts/sysroot-relativelinks.py b/scripts/sysroot-relativelinks.py new file mode 100755 index 0000000..ac26367 --- /dev/null +++ b/scripts/sysroot-relativelinks.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +import sys +import os + +# Take a sysroot directory and turn all the abolute symlinks and turn them into +# relative ones such that the sysroot is usable within another system. + +if len(sys.argv) != 2: + print("Usage is " + sys.argv[0] + "<directory>") + sys.exit(1) + +topdir = sys.argv[1] +topdir = os.path.abspath(topdir) + +def handlelink(filep, subdir): + link = os.readlink(filep) + if link[0] != "/": + return + if link.startswith(topdir): + return + #print("Replacing %s with %s for %s" % (link, topdir+link, filep)) + print("Replacing %s with %s for %s" % (link, os.path.relpath(topdir+link, subdir), filep)) + os.unlink(filep) + os.symlink(os.path.relpath(topdir+link, subdir), filep) + +for subdir, dirs, files in os.walk(topdir): + for f in files: + filep = os.path.join(subdir, f) + if os.path.islink(filep): + #print("Considering %s" % filep) + handlelink(filep, subdir) diff --git a/scripts/test-dependencies.sh b/scripts/test-dependencies.sh new file mode 100755 index 0000000..0170947 --- /dev/null +++ b/scripts/test-dependencies.sh @@ -0,0 +1,286 @@ +#!/bin/bash + +# Author: Martin Jansa <martin.jansa@gmail.com> +# +# Copyright (c) 2013 Martin Jansa <Martin.Jansa@gmail.com> + +# Used to detect missing dependencies or automagically +# enabled dependencies which aren't explicitly enabled +# or disabled. Using bash to have PIPESTATUS variable. + +# It does 3 builds of <target> +# 1st to populate sstate-cache directory and sysroot +# 2nd to rebuild each recipe with every possible +# dependency found in sysroot (which stays populated +# from 1st build +# 3rd to rebuild each recipe only with dependencies defined +# in DEPENDS +# 4th (optional) repeat build like 3rd to make sure that +# minimal versions of dependencies defined in DEPENDS +# is also enough + +# Global vars +tmpdir= +targets= +recipes= +buildhistory= +buildtype= +default_targets="world" +default_buildhistory="buildhistory" +default_buildtype="1 2 3 c" + +usage () { + cat << EOF +Welcome to utility to detect missing or autoenabled dependencies. +WARNING: this utility will completely remove your tmpdir (make sure + you don't have important buildhistory or persistent dir there). +$0 <OPTION> + +Options: + -h, --help + Display this help and exit. + + --tmpdir=<tmpdir> + Specify tmpdir, will use the environment variable TMPDIR if it is not specified. + Something like /OE/oe-core/tmp-eglibc (no / at the end). + + --targets=<targets> + List of targets separated by space, will use the environment variable TARGETS if it is not specified. + It will run "bitbake <targets>" to populate sysroots. + Default value is "world". + + --recipes=<recipes> + File with list of recipes we want to rebuild with minimal and maximal sysroot. + Will use the environment variable RECIPES if it is not specified. + Default value will use all packages ever recorded in buildhistory directory. + + --buildhistory=<buildhistory> + Path to buildhistory directory, it needs to be enabled in your config, + because it's used to detect different dependencies and to create list + of recipes to rebuild when it's not specified. + Will use the environment variable BUILDHISTORY if it is not specified. + Default value is "buildhistory" + + --buildtype=<buildtype> + There are 4 types of build: + 1: build to populate sstate-cache directory and sysroot + 2: build to rebuild each recipe with every possible dep + 3: build to rebuild each recipe with minimal dependencies + 4: build to rebuild each recipe again with minimal dependencies + c: compare buildhistory directories from build 2 and 3 + Will use the environment variable BUILDTYPE if it is not specified. + Default value is "1 2 3 c", order is important, type 4 is optional. +EOF +} + +# Print error information and exit. +echo_error () { + echo "ERROR: $1" >&2 + exit 1 +} + +while [ -n "$1" ]; do + case $1 in + --tmpdir=*) + tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e` + [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir" + shift + ;; + --targets=*) + targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'` + shift + ;; + --recipes=*) + recipes=`echo $1 | sed -e 's#^--recipes="*\([^"]*\)"*#\1#'` + shift + ;; + --buildhistory=*) + buildhistory=`echo $1 | sed -e 's#^--buildhistory="*\([^"]*\)"*#\1#'` + shift + ;; + --buildtype=*) + buildtype=`echo $1 | sed -e 's#^--buildtype="*\([^"]*\)"*#\1#'` + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Invalid arguments $*" + echo_error "Try '$0 -h' for more information." + ;; + esac +done + +# tmpdir directory, use environment variable TMPDIR +# if it was not specified, otherwise, error. +[ -n "$tmpdir" ] || tmpdir=$TMPDIR +[ -n "$tmpdir" ] || echo_error "No tmpdir found!" +[ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\"" +[ -n "$targets" ] || targets=$TARGETS +[ -n "$targets" ] || targets=$default_targets +[ -n "$recipes" ] || recipes=$RECIPES +[ -n "$recipes" -a ! -f "$recipes" ] && echo_error "Invalid file with list of recipes to rebuild" +[ -n "$recipes" ] || echo "All packages ever recorded in buildhistory directory will be rebuilt" +[ -n "$buildhistory" ] || buildhistory=$BUILDHISTORY +[ -n "$buildhistory" ] || buildhistory=$default_buildhistory +[ -d "$buildhistory" ] || echo_error "Invalid buildhistory directory \"$buildhistory\"" +[ -n "$buildtype" ] || buildtype=$BUILDTYPE +[ -n "$buildtype" ] || buildtype=$default_buildtype +echo "$buildtype" | grep -v '^[1234c ]*$' && echo_error "Invalid buildtype \"$buildtype\", only some combination of 1, 2, 3, 4, c separated by space is allowed" + +OUTPUT_BASE=test-dependencies/`date "+%s"` +declare -i RESULT=0 + +build_all() { + echo "===== 1st build to populate sstate-cache directory and sysroot =====" + OUTPUT1=${OUTPUT_BASE}/${TYPE}_all + mkdir -p ${OUTPUT1} + echo "Logs will be stored in ${OUTPUT1} directory" + bitbake -k $targets 2>&1 | tee -a ${OUTPUT1}/complete.log + RESULT+=${PIPESTATUS[0]} + grep "ERROR: Task.*failed" ${OUTPUT1}/complete.log > ${OUTPUT1}/failed-tasks.log + cat ${OUTPUT1}/failed-tasks.log | sed 's@.*/@@g; s@_.*@@g; s@\.bb, .*@@g' | sort -u > ${OUTPUT1}/failed-recipes.log +} + +build_every_recipe() { + if [ "${TYPE}" = "2" ] ; then + echo "===== 2nd build to rebuild each recipe with every possible dep =====" + OUTPUT_MAX=${OUTPUT_BASE}/${TYPE}_max + OUTPUTB=${OUTPUT_MAX} + else + echo "===== 3rd or 4th build to rebuild each recipe with minimal dependencies =====" + OUTPUT_MIN=${OUTPUT_BASE}/${TYPE}_min + OUTPUTB=${OUTPUT_MIN} + fi + + mkdir -p ${OUTPUTB} ${OUTPUTB}/failed ${OUTPUTB}/ok + echo "Logs will be stored in ${OUTPUTB} directory" + if [ -z "$recipes" ]; then + ls -d $buildhistory/packages/*/* | xargs -n 1 basename | sort -u > ${OUTPUTB}/recipe.list + recipes=${OUTPUTB}/recipe.list + fi + if [ "${TYPE}" != "2" ] ; then + echo "!!!Removing tmpdir \"$tmpdir\"!!!" + rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null + fi + i=1 + count=`cat $recipes ${OUTPUT1}/failed-recipes.log | sort -u | wc -l` + for recipe in `cat $recipes ${OUTPUT1}/failed-recipes.log | sort -u`; do + echo "Building recipe: ${recipe} ($i/$count)" + declare -i RECIPE_RESULT=0 + bitbake -c cleansstate ${recipe} > ${OUTPUTB}/${recipe}.log 2>&1; + RECIPE_RESULT+=$? + bitbake ${recipe} >> ${OUTPUTB}/${recipe}.log 2>&1; + RECIPE_RESULT+=$? + if [ "${RECIPE_RESULT}" != "0" ] ; then + RESULT+=${RECIPE_RESULT} + mv ${OUTPUTB}/${recipe}.log ${OUTPUTB}/failed/ + grep "ERROR: Task.*failed" ${OUTPUTB}/failed/${recipe}.log | tee -a ${OUTPUTB}/failed-tasks.log + grep "ERROR: Task.*failed" ${OUTPUTB}/failed/${recipe}.log | sed 's@.*/@@g; s@_.*@@g; s@\.bb, .*@@g' >> ${OUTPUTB}/failed-recipes.log + # and append also ${recipe} in case the failed task was from some dependency + echo ${recipe} >> ${OUTPUTB}/failed-recipes.log + else + mv ${OUTPUTB}/${recipe}.log ${OUTPUTB}/ok/ + fi + if [ "${TYPE}" != "2" ] ; then + rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null + fi + i=`expr $i + 1` + done + echo "Copying buildhistory/packages to ${OUTPUTB}" + cp -ra $buildhistory/packages ${OUTPUTB} + # This will be usefull to see which library is pulling new dependency + echo "Copying do_package logs to ${OUTPUTB}/do_package/" + mkdir ${OUTPUTB}/do_package + find $tmpdir/work/ -name log.do_package 2>/dev/null| while read f; do + # pn is 3 levels back, but we don't know if there is just one log per pn (only one arch and version) + # dest=`echo $f | sed 's#^.*/\([^/]*\)/\([^/]*\)/\([^/]*\)/log.do_package#\1#g'` + dest=`echo $f | sed "s#$tmpdir/work/##g; s#/#_#g"` + cp $f ${OUTPUTB}/do_package/$dest + done +} + +compare_deps() { + # you can run just compare task with command like this + # OUTPUT_BASE=test-dependencies/1373140172 \ + # OUTPUT_MAX=${OUTPUT_BASE}/2_max \ + # OUTPUT_MIN=${OUTPUT_BASE}/3_min \ + # openembedded-core/scripts/test-dependencies.sh --tmpdir=tmp-eglibc --targets=glib-2.0 --recipes=recipe_list --buildtype=c + echo "===== Compare dependencies recorded in \"${OUTPUT_MAX}\" and \"${OUTPUT_MIN}\" =====" + [ -n "${OUTPUTC}" ] || OUTPUTC=${OUTPUT_BASE}/comp + mkdir -p ${OUTPUTC} + OUTPUT_FILE=${OUTPUTC}/dependency-changes + echo "Differences will be stored in ${OUTPUT_FILE}, dot is shown for every 100 of checked packages" + echo > ${OUTPUT_FILE} + + [ -d ${OUTPUT_MAX} ] || echo_error "Directory with output from build 2 \"${OUTPUT_MAX}\" does not exist" + [ -d ${OUTPUT_MIN} ] || echo_error "Directory with output from build 3 \"${OUTPUT_MIN}\" does not exist" + [ -d ${OUTPUT_MAX}/packages/ ] || echo_error "Directory with packages from build 2 \"${OUTPUT_MAX}/packages/\" does not exist" + [ -d ${OUTPUT_MIN}/packages/ ] || echo_error "Directory with packages from build 3 \"${OUTPUT_MIN}/packages/\" does not exist" + i=0 + find ${OUTPUT_MAX}/packages/ -name latest | sed "s#${OUTPUT_MAX}/##g" | while read pkg; do + max_pkg=${OUTPUT_MAX}/${pkg} + min_pkg=${OUTPUT_MIN}/${pkg} + # pkg=packages/armv5te-oe-linux-gnueabi/libungif/libungif/latest + recipe=`echo "${pkg}" | sed 's#packages/[^/]*/\([^/]*\)/\([^/]*\)/latest#\1#g'` + package=`echo "${pkg}" | sed 's#packages/[^/]*/\([^/]*\)/\([^/]*\)/latest#\2#g'` + if [ ! -f "${min_pkg}" ] ; then + echo "ERROR: ${recipe}: ${package} package isn't created when building with minimal dependencies?" | tee -a ${OUTPUT_FILE} + echo ${recipe} >> ${OUTPUTC}/failed-recipes.log + continue + fi + # strip version information in parenthesis + max_deps=`grep "^RDEPENDS = " ${max_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'` + min_deps=`grep "^RDEPENDS = " ${min_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'` + if [ "$i" = 100 ] ; then + echo -n "." # cheap progressbar + i=0 + fi + if [ "${max_deps}" = "${min_deps}" ] ; then + # it's annoying long, but at least it's showing some progress, warnings are grepped at the end + echo "NOTE: ${recipe}: ${package} rdepends weren't changed" >> ${OUTPUT_FILE} + else + missing_deps= + for dep in ${max_deps}; do + if ! echo "${min_deps}" | grep -q " ${dep} " ; then + missing_deps="${missing_deps} ${dep}" + echo # to get rid of dots on last line + echo "WARN: ${recipe}: ${package} rdepends on ${dep}, but it isn't a build dependency?" | tee -a ${OUTPUT_FILE} + fi + done + if [ -n "${missing_deps}" ] ; then + echo ${recipe} >> ${OUTPUTC}/failed-recipes.log + fi + fi + i=`expr $i + 1` + done + echo # to get rid of dots on last line + echo "Found differences: " + grep "^WARN: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.warn.log + echo "Found errors: " + grep "^ERROR: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.error.log + RESULT+=`cat ${OUTPUT_FILE}.warn.log | wc -l` + RESULT+=`cat ${OUTPUT_FILE}.error.log | wc -l` +} + +for TYPE in $buildtype; do + case ${TYPE} in + 1) build_all;; + 2) build_every_recipe;; + 3) build_every_recipe;; + 4) build_every_recipe;; + c) compare_deps;; + *) echo_error "Invalid buildtype \"$TYPE\"" + esac +done + +cat ${OUTPUT_BASE}/*/failed-recipes.log | sort -u >> ${OUTPUT_BASE}/failed-recipes.log + +if [ "${RESULT}" != "0" ] ; then + echo "ERROR: ${RESULT} issues were found in these recipes: `cat ${OUTPUT_BASE}/failed-recipes.log | xargs`" +fi + +echo "INFO: Output written in: ${OUTPUT_BASE}" +exit ${RESULT} diff --git a/scripts/test-reexec b/scripts/test-reexec new file mode 100755 index 0000000..9eaa96e --- /dev/null +++ b/scripts/test-reexec @@ -0,0 +1,123 @@ +#!/bin/bash + +# Test Script for task re-execution +# +# Copyright 2012 Intel Corporation +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# DESCRIPTION +# This script is intended to address issues for re-execution of +# tasks. The test results are saved in ./reexeclogs. Force build +# logs are saved with prefix "force". Build failure logs are saved with +# prefix "failed". Log files with prefix "initial" are used to save +# initial build logs for each recipe. Log files with prefix "clean" are +# used to save logs of clean task after testing for a recipe is finished. +# + +targets=`bitbake -s | cut -d " " -f 1` + +LOGS=./reexeclogs + +mkdir -p $LOGS + +# Clear sstate files for specified recipe +function clearsstate { + target=$1 + + sstate_dir=`bitbake $target -e | grep "^SSTATE_DIR" | cut -d "\"" -f 2` + sstate_pkgspec=`bitbake $target -e | grep "^SSTATE_PKGSPEC" | cut -d "\"" -f 2` + sstasks=`bitbake $target -e | grep "^SSTATETASKS" | cut -d "\"" -f 2` + + for sstask in $sstasks + do + sstask=${sstask:3} + case $sstask in + populate_sysroot) sstask="populate-sysroot" + ;; + populate_lic) sstask="populate-lic" + ;; + package_write_ipk) sstask="deploy-ipk" + ;; + package_write_deb) sstask="deploy-deb" + ;; + package_write_rpm) sstask="deploy-rpm" + ;; + package) sstask="package" + ;; + deploy) sstask="deploy" + ;; + *) + ;; + esac + + echo "Removing ${sstate_dir}/${sstate_pkgspec}*_${sstask}.tgz* for $target" + rm -rf ${sstate_dir}/${sstate_pkgspec}*_${sstask}.tgz* + done +} + +# Function to re-execute specified task of recipe +function testit { + target=$1 + task=$2 + + task=`echo $task | sed 's/_setscene//'` + + if [ -f $LOGS/force.$target.$task ]; then + return + fi + + case $task in + clean|build|cleansstate|cleanall|package|cleansstate2|package_write|package_write_ipk|package_write_rpm|package_write_deb|fetch|populate_lic) return;; + fetchall|devshell|buildall|listtasks|checkuri|checkuriall) return;; + esac + + echo "Attempting target $target, task $task" + echo "Initial build" + bitbake $target -c cleansstate > $LOGS/initial.$target.$task + bitbake $target >> $LOGS/initial.$target.$task + clearsstate $target >> $LOGS/initial.$target.$task + echo "Re-execution build" + bitbake $target -c $task -f > $LOGS/force.$target.$task + if [ "$?" != 0 ]; then + echo "FAILURE for $target $task" + cp $LOGS/force.$target.$task $LOGS/failed.$target.$task + bitbake $target -c clean > $LOGS/clean.$target.$task + else + bitbake $target >> $LOGS/force.$target.$task + if [ "$?" != 0 ]; then + echo "FAILURE2 for $target $task" + cp $LOGS/force.$target.$task $LOGS/failed.$target.$task + bitbake $target -c clean > $LOGS/clean.$target.$task + fi + fi + echo "Done" +} + +# Go through the recipe list and these recipes' task list +# Then re-execute them +for target in $targets; do + # Remove log messages from bitbake output + case $target in + Summary*|WARNING*|Loading*|Loaded*|Package*|=====*) continue;; + esac + tasks=`bitbake $target -c listtasks | grep ^do_ | sed s/do_//` + for task in $tasks; do + testit $target $task + done +done + + diff --git a/scripts/test-remote-image b/scripts/test-remote-image new file mode 100755 index 0000000..9c5b015 --- /dev/null +++ b/scripts/test-remote-image @@ -0,0 +1,361 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# DESCRIPTION +# This script is used to test public autobuilder images on remote hardware. +# The script is called from a machine that is able download the images from the remote images repository and to connect to the test hardware. +# +# test-remote-image --image-type core-image-sato --repo-link http://192.168.10.2/images --required-packages rpm psplash +# +# Translation: Build the 'rpm' and 'pslash' packages and test a remote core-image-sato image using the http://192.168.10.2/images repository. +# +# You can also use the '-h' option to see some help information. + +import os +import sys +import argparse +import logging +import shutil +from abc import ABCMeta, abstractmethod + +# Add path to scripts/lib in sys.path; +scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] + +import scriptpath +import argparse_oe + +# Add meta/lib to sys.path +scriptpath.add_oe_lib_path() + +import oeqa.utils.ftools as ftools +from oeqa.utils.commands import runCmd, bitbake, get_bb_var + +# Add all lib paths relative to BBPATH to sys.path; this is used to find and import the target controllers. +for path in get_bb_var('BBPATH').split(":"): + sys.path.insert(0, os.path.abspath(os.path.join(path, 'lib'))) + +# In order to import modules that contain target controllers, we need the bitbake libraries in sys.path . +bitbakepath = scriptpath.add_bitbake_lib_path() +if not bitbakepath: + sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") + sys.exit(1) + +# create a logger +def logger_create(): + log = logging.getLogger('hwauto') + log.setLevel(logging.DEBUG) + + fh = logging.FileHandler(filename='hwauto.log', mode='w') + fh.setLevel(logging.DEBUG) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + ch.setFormatter(formatter) + + log.addHandler(fh) + log.addHandler(ch) + + return log + +# instantiate the logger +log = logger_create() + + +# Define and return the arguments parser for the script +def get_args_parser(): + description = "This script is used to run automated runtime tests using remotely published image files. You should prepare the build environment just like building local images and running the tests." + parser = argparse_oe.ArgumentParser(description=description) + parser.add_argument('--image-types', required=True, action="store", nargs='*', dest="image_types", default=None, help='The image types to test(ex: core-image-minimal).') + parser.add_argument('--repo-link', required=True, action="store", type=str, dest="repo_link", default=None, help='The link to the remote images repository.') + parser.add_argument('--required-packages', required=False, action="store", nargs='*', dest="required_packages", default=None, help='Required packages for the tests. They will be built before the testing begins.') + parser.add_argument('--targetprofile', required=False, action="store", nargs=1, dest="targetprofile", default='AutoTargetProfile', help='The target profile to be used.') + parser.add_argument('--repoprofile', required=False, action="store", nargs=1, dest="repoprofile", default='PublicAB', help='The repo profile to be used.') + parser.add_argument('--skip-download', required=False, action="store_true", dest="skip_download", default=False, help='Skip downloading the images completely. This needs the correct files to be present in the directory specified by the target profile.') + return parser + +class BaseTargetProfile(object): + """ + This class defines the meta profile for a specific target (MACHINE type + image type). + """ + + __metaclass__ = ABCMeta + + def __init__(self, image_type): + self.image_type = image_type + + self.kernel_file = None + self.rootfs_file = None + self.manifest_file = None + self.extra_download_files = [] # Extra files (full name) to be downloaded. They should be situated in repo_link + + # This method is used as the standard interface with the target profile classes. + # It returns a dictionary containing a list of files and their meaning/description. + def get_files_dict(self): + files_dict = {} + + if self.kernel_file: + files_dict['kernel_file'] = self.kernel_file + else: + log.error('The target profile did not set a kernel file.') + sys.exit(1) + + if self.rootfs_file: + files_dict['rootfs_file'] = self.rootfs_file + else: + log.error('The target profile did not set a rootfs file.') + sys.exit(1) + + if self.manifest_file: + files_dict['manifest_file'] = self.manifest_file + else: + log.error('The target profile did not set a manifest file.') + sys.exit(1) + + for idx, f in enumerate(self.extra_download_files): + files_dict['extra_download_file' + str(idx)] = f + + return files_dict + +class AutoTargetProfile(BaseTargetProfile): + + def __init__(self, image_type): + super(AutoTargetProfile, self).__init__(image_type) + self.image_name = get_bb_var('IMAGE_LINK_NAME', target=image_type) + self.kernel_type = get_bb_var('KERNEL_IMAGETYPE', target=image_type) + self.controller = self.get_controller() + + self.set_kernel_file() + self.set_rootfs_file() + self.set_manifest_file() + self.set_extra_download_files() + + # Get the controller object that will be used by bitbake. + def get_controller(self): + from oeqa.controllers.testtargetloader import TestTargetLoader + + target_controller = get_bb_var('TEST_TARGET') + bbpath = get_bb_var('BBPATH').split(':') + + if target_controller == "qemu": + from oeqa.targetcontrol import QemuTarget + controller = QemuTarget + else: + testtargetloader = TestTargetLoader() + controller = testtargetloader.get_controller_module(target_controller, bbpath) + return controller + + def set_kernel_file(self): + postconfig = "QA_GET_MACHINE = \"${MACHINE}\"" + machine = get_bb_var('QA_GET_MACHINE', postconfig=postconfig) + self.kernel_file = self.kernel_type + '-' + machine + '.bin' + + def set_rootfs_file(self): + image_fstypes = get_bb_var('IMAGE_FSTYPES').split(' ') + # Get a matching value between target's IMAGE_FSTYPES and the image fstypes suppoerted by the target controller. + fstype = self.controller.match_image_fstype(d=None, image_fstypes=image_fstypes) + if fstype: + self.rootfs_file = self.image_name + '.' + fstype + else: + log.error("Could not get a compatible image fstype. Check that IMAGE_FSTYPES and the target controller's supported_image_fstypes fileds have common values.") + sys.exit(1) + + def set_manifest_file(self): + self.manifest_file = self.image_name + ".manifest" + + def set_extra_download_files(self): + self.extra_download_files = self.get_controller_extra_files() + if not self.extra_download_files: + self.extra_download_files = [] + + def get_controller_extra_files(self): + controller = self.get_controller() + return controller.get_extra_files() + + +class BaseRepoProfile(object): + """ + This class defines the meta profile for an images repository. + """ + + __metaclass__ = ABCMeta + + def __init__(self, repolink, localdir): + self.localdir = localdir + self.repolink = repolink + + # The following abstract methods are the interfaces to the repository profile classes derived from this abstract class. + + # This method should check the file named 'file_name' if it is different than the upstream one. + # Should return False if the image is the same as the upstream and True if it differs. + @abstractmethod + def check_old_file(self, file_name): + pass + + # This method should fetch file_name and create a symlink to localname if set. + @abstractmethod + def fetch(self, file_name, localname=None): + pass + +class PublicAB(BaseRepoProfile): + + def __init__(self, repolink, localdir=None): + super(PublicAB, self).__init__(repolink, localdir) + if localdir is None: + self.localdir = os.path.join(os.environ['BUILDDIR'], 'PublicABMirror') + + # Not yet implemented. Always returning True. + def check_old_file(self, file_name): + return True + + def get_repo_path(self): + path = '/machines/' + + postconfig = "QA_GET_MACHINE = \"${MACHINE}\"" + machine = get_bb_var('QA_GET_MACHINE', postconfig=postconfig) + if 'qemu' in machine: + path += 'qemu/' + + postconfig = "QA_GET_DISTRO = \"${DISTRO}\"" + distro = get_bb_var('QA_GET_DISTRO', postconfig=postconfig) + path += distro.replace('poky', machine) + '/' + return path + + + def fetch(self, file_name, localname=None): + repo_path = self.get_repo_path() + link = self.repolink + repo_path + file_name + + self.wget(link, self.localdir, localname) + + def wget(self, link, localdir, localname=None, extraargs=None): + wget_cmd = '/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate ' + + if localname: + wget_cmd += ' -O ' + localname + ' ' + + if extraargs: + wget_cmd += ' ' + extraargs + ' ' + + wget_cmd += " -P %s '%s'" % (localdir, link) + runCmd(wget_cmd) + +class HwAuto(): + + def __init__(self, image_types, repolink, required_packages, targetprofile, repoprofile, skip_download): + log.info('Initializing..') + self.image_types = image_types + self.repolink = repolink + self.required_packages = required_packages + self.targetprofile = targetprofile + self.repoprofile = repoprofile + self.skip_download = skip_download + self.repo = self.get_repo_profile(self.repolink) + + # Get the repository profile; for now we only look inside this module. + def get_repo_profile(self, *args, **kwargs): + repo = getattr(sys.modules[__name__], self.repoprofile)(*args, **kwargs) + log.info("Using repo profile: %s" % repo.__class__.__name__) + return repo + + # Get the target profile; for now we only look inside this module. + def get_target_profile(self, *args, **kwargs): + target = getattr(sys.modules[__name__], self.targetprofile)(*args, **kwargs) + log.info("Using target profile: %s" % target.__class__.__name__) + return target + + # Run the testimage task on a build while redirecting DEPLOY_DIR_IMAGE to repo.localdir, where the images are downloaded. + def runTestimageBuild(self, image_type): + log.info("Running the runtime tests for %s.." % image_type) + postconfig = "DEPLOY_DIR_IMAGE = \"%s\"" % self.repo.localdir + result = bitbake("%s -c testimage" % image_type, ignore_status=True, postconfig=postconfig) + testimage_results = ftools.read_file(os.path.join(get_bb_var("T", image_type), "log.do_testimage")) + log.info('Runtime tests results for %s:' % image_type) + print testimage_results + return result + + # Start the procedure! + def run(self): + if self.required_packages: + # Build the required packages for the tests + log.info("Building the required packages: %s ." % ', '.join(map(str, self.required_packages))) + result = bitbake(self.required_packages, ignore_status=True) + if result.status != 0: + log.error("Could not build required packages: %s. Output: %s" % (self.required_packages, result.output)) + sys.exit(1) + + # Build the package repository meta data. + log.info("Building the package index.") + result = bitbake("package-index", ignore_status=True) + if result.status != 0: + log.error("Could not build 'package-index'. Output: %s" % result.output) + sys.exit(1) + + # Create the directory structure for the images to be downloaded + log.info("Creating directory structure %s" % self.repo.localdir) + if not os.path.exists(self.repo.localdir): + os.makedirs(self.repo.localdir) + + # For each image type, download the needed files and run the tests. + noissuesfound = True + for image_type in self.image_types: + if self.skip_download: + log.info("Skipping downloading the images..") + else: + target = self.get_target_profile(image_type) + files_dict = target.get_files_dict() + log.info("Downloading files for %s" % image_type) + for f in files_dict: + if self.repo.check_old_file(files_dict[f]): + filepath = os.path.join(self.repo.localdir, files_dict[f]) + if os.path.exists(filepath): + os.remove(filepath) + self.repo.fetch(files_dict[f]) + + result = self.runTestimageBuild(image_type) + if result.status != 0: + noissuesfound = False + + if noissuesfound: + log.info('Finished. No issues found.') + else: + log.error('Finished. Some runtime tests have failed. Returning non-0 status code.') + sys.exit(1) + + + +def main(): + + parser = get_args_parser() + args = parser.parse_args() + + hwauto = HwAuto(image_types=args.image_types, repolink=args.repo_link, required_packages=args.required_packages, targetprofile=args.targetprofile, repoprofile=args.repoprofile, skip_download=args.skip_download) + + hwauto.run() + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/scripts/tiny/dirsize.py b/scripts/tiny/dirsize.py new file mode 100755 index 0000000..40ff4ab --- /dev/null +++ b/scripts/tiny/dirsize.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# +# Display details of the root filesystem size, broken up by directory. +# Allows for limiting by size to focus on the larger files. +# +# Author: Darren Hart <dvhart@linux.intel.com> +# + +import os +import sys +import stat + +class Record: + def create(path): + r = Record(path) + + s = os.lstat(path) + if stat.S_ISDIR(s.st_mode): + for p in os.listdir(path): + pathname = path + "/" + p + ss = os.lstat(pathname) + if not stat.S_ISLNK(ss.st_mode): + r.records.append(Record.create(pathname)) + r.size += r.records[-1].size + r.records.sort(reverse=True) + else: + r.size = os.lstat(path).st_size + + return r + create = staticmethod(create) + + def __init__(self, path): + self.path = path + self.size = 0 + self.records = [] + + def __cmp__(this, that): + if that is None: + return 1 + if not isinstance(that, Record): + raise TypeError + if len(this.records) > 0 and len(that.records) == 0: + return -1 + if len(this.records) == 0 and len(that.records) > 0: + return 1 + if this.size < that.size: + return -1 + if this.size > that.size: + return 1 + return 0 + + def show(self, minsize): + total = 0 + if self.size <= minsize: + return 0 + print "%10d %s" % (self.size, self.path) + for r in self.records: + total += r.show(minsize) + if len(self.records) == 0: + total = self.size + return total + + +def main(): + minsize = 0 + if len(sys.argv) == 2: + minsize = int(sys.argv[1]) + rootfs = Record.create(".") + total = rootfs.show(minsize) + print "Displayed %d/%d bytes (%.2f%%)" % \ + (total, rootfs.size, 100 * float(total) / rootfs.size) + + +if __name__ == "__main__": + main() diff --git a/scripts/tiny/ksize.py b/scripts/tiny/ksize.py new file mode 100755 index 0000000..4006f2f --- /dev/null +++ b/scripts/tiny/ksize.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# +# Display details of the kernel build size, broken up by built-in.o. Sort +# the objects by size. Run from the top level kernel build directory. +# +# Author: Darren Hart <dvhart@linux.intel.com> +# + +import sys +import getopt +import os +from subprocess import * +from string import join + + +def usage(): + prog = os.path.basename(sys.argv[0]) + print 'Usage: %s [OPTION]...' % (prog) + print ' -d, display an additional level of drivers detail' + print ' -h, --help display this help and exit' + print '' + print 'Run %s from the top-level Linux kernel build directory.' % (prog) + + +class Sizes: + def __init__(self, glob): + self.title = glob + p = Popen("size -t " + glob, shell=True, stdout=PIPE, stderr=PIPE) + output = p.communicate()[0].splitlines() + if len(output) > 2: + sizes = output[-1].split()[0:4] + self.text = int(sizes[0]) + self.data = int(sizes[1]) + self.bss = int(sizes[2]) + self.total = int(sizes[3]) + else: + self.text = self.data = self.bss = self.total = 0 + + def show(self, indent=""): + print "%-32s %10d | %10d %10d %10d" % \ + (indent+self.title, self.total, self.text, self.data, self.bss) + + +class Report: + def create(filename, title, subglob=None): + r = Report(filename, title) + path = os.path.dirname(filename) + + p = Popen("ls " + path + "/*.o | grep -v built-in.o", + shell=True, stdout=PIPE, stderr=PIPE) + glob = join(p.communicate()[0].splitlines()) + oreport = Report(glob, path + "/*.o") + oreport.sizes.title = path + "/*.o" + r.parts.append(oreport) + + if subglob: + p = Popen("ls " + subglob, shell=True, stdout=PIPE, stderr=PIPE) + for f in p.communicate()[0].splitlines(): + path = os.path.dirname(f) + r.parts.append(Report.create(f, path, path + "/*/built-in.o")) + r.parts.sort(reverse=True) + + for b in r.parts: + r.totals["total"] += b.sizes.total + r.totals["text"] += b.sizes.text + r.totals["data"] += b.sizes.data + r.totals["bss"] += b.sizes.bss + + r.deltas["total"] = r.sizes.total - r.totals["total"] + r.deltas["text"] = r.sizes.text - r.totals["text"] + r.deltas["data"] = r.sizes.data - r.totals["data"] + r.deltas["bss"] = r.sizes.bss - r.totals["bss"] + return r + create = staticmethod(create) + + def __init__(self, glob, title): + self.glob = glob + self.title = title + self.sizes = Sizes(glob) + self.parts = [] + self.totals = {"total":0, "text":0, "data":0, "bss":0} + self.deltas = {"total":0, "text":0, "data":0, "bss":0} + + def show(self, indent=""): + rule = str.ljust(indent, 80, '-') + print "%-32s %10s | %10s %10s %10s" % \ + (indent+self.title, "total", "text", "data", "bss") + print rule + self.sizes.show(indent) + print rule + for p in self.parts: + if p.sizes.total > 0: + p.sizes.show(indent) + print rule + print "%-32s %10d | %10d %10d %10d" % \ + (indent+"sum", self.totals["total"], self.totals["text"], + self.totals["data"], self.totals["bss"]) + print "%-32s %10d | %10d %10d %10d" % \ + (indent+"delta", self.deltas["total"], self.deltas["text"], + self.deltas["data"], self.deltas["bss"]) + print "\n" + + def __cmp__(this, that): + if that is None: + return 1 + if not isinstance(that, Report): + raise TypeError + if this.sizes.total < that.sizes.total: + return -1 + if this.sizes.total > that.sizes.total: + return 1 + return 0 + + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], "dh", ["help"]) + except getopt.GetoptError, err: + print '%s' % str(err) + usage() + sys.exit(2) + + driver_detail = False + for o, a in opts: + if o == '-d': + driver_detail = True + elif o in ('-h', '--help'): + usage() + sys.exit(0) + else: + assert False, "unhandled option" + + glob = "arch/*/built-in.o */built-in.o" + vmlinux = Report.create("vmlinux", "Linux Kernel", glob) + + vmlinux.show() + for b in vmlinux.parts: + if b.totals["total"] > 0 and len(b.parts) > 1: + b.show() + if b.title == "drivers" and driver_detail: + for d in b.parts: + if d.totals["total"] > 0 and len(d.parts) > 1: + d.show(" ") + + +if __name__ == "__main__": + main() diff --git a/scripts/wic b/scripts/wic new file mode 100755 index 0000000..2286f20 --- /dev/null +++ b/scripts/wic @@ -0,0 +1,323 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION 'wic' is the OpenEmbedded Image Creator that users can +# use to generate bootable images. Invoking it without any arguments +# will display help screens for the 'wic' command and list the +# available 'wic' subcommands. Invoking a subcommand without any +# arguments will likewise display help screens for the specified +# subcommand. Please use that interface for detailed help. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# + +__version__ = "0.2.0" + +# Python Standard Library modules +import os +import sys +import optparse +import logging +from distutils import spawn + +# External modules +scripts_path = os.path.abspath(os.path.dirname(__file__)) +lib_path = scripts_path + '/lib' +sys.path.insert(0, lib_path) + +bitbake_exe = spawn.find_executable('bitbake') +if bitbake_exe: + bitbake_path = os.path.join(os.path.dirname(bitbake_exe), '../lib') + sys.path.insert(0, bitbake_path) + from bb import cookerdata + from bb.main import bitbake_main, BitBakeConfigParameters +else: + bitbake_main = None + +from wic.utils.oe.misc import get_bitbake_var, BB_VARS +from wic.utils.errors import WicError +from wic import engine +from wic import help as hlp + +def rootfs_dir_to_args(krootfs_dir): + """ + Get a rootfs_dir dict and serialize to string + """ + rootfs_dir = '' + for key, val in krootfs_dir.items(): + rootfs_dir += ' ' + rootfs_dir += '='.join([key, val]) + return rootfs_dir.strip() + +def callback_rootfs_dir(option, opt, value, parser): + """ + Build a dict using --rootfs_dir connection=dir + """ + if not type(parser.values.rootfs_dir) is dict: + parser.values.rootfs_dir = dict() + + if '=' in value: + (key, rootfs_dir) = value.split('=') + else: + key = 'ROOTFS_DIR' + rootfs_dir = value + + parser.values.rootfs_dir[key] = rootfs_dir + +def wic_create_subcommand(args, usage_str): + """ + Command-line handling for image creation. The real work is done + by image.engine.wic_create() + """ + parser = optparse.OptionParser(usage=usage_str) + + parser.add_option("-o", "--outdir", dest="outdir", + help="name of directory to create image in") + parser.add_option("-e", "--image-name", dest="image_name", + help="name of the image to use the artifacts from " + "e.g. core-image-sato") + parser.add_option("-r", "--rootfs-dir", dest="rootfs_dir", type="string", + action="callback", callback=callback_rootfs_dir, + help="path to the /rootfs dir to use as the " + ".wks rootfs source") + parser.add_option("-b", "--bootimg-dir", dest="bootimg_dir", + help="path to the dir containing the boot artifacts " + "(e.g. /EFI or /syslinux dirs) to use as the " + ".wks bootimg source") + parser.add_option("-k", "--kernel-dir", dest="kernel_dir", + help="path to the dir containing the kernel to use " + "in the .wks bootimg") + parser.add_option("-n", "--native-sysroot", dest="native_sysroot", + help="path to the native sysroot containing the tools " + "to use to build the image") + parser.add_option("-p", "--skip-build-check", dest="build_check", + action="store_false", default=True, help="skip the build check") + parser.add_option("-f", "--build-rootfs", action="store_true", help="build rootfs") + parser.add_option("-c", "--compress-with", choices=("gzip", "bzip2", "xz"), + dest='compressor', + help="compress image with specified compressor") + parser.add_option("-v", "--vars", dest='vars_dir', + help="directory with <image>.env files that store " + "bitbake variables") + parser.add_option("-D", "--debug", dest="debug", action="store_true", + default=False, help="output debug information") + + (options, args) = parser.parse_args(args) + + if len(args) != 1: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + if options.build_rootfs and not bitbake_main: + logging.error("Can't build roofs as bitbake is not in the $PATH") + sys.exit(1) + + if not options.image_name: + missed = [] + for val, opt in [(options.rootfs_dir, 'rootfs-dir'), + (options.bootimg_dir, 'bootimg-dir'), + (options.kernel_dir, 'kernel-dir'), + (options.native_sysroot, 'native-sysroot')]: + if not val: + missed.append(opt) + if missed: + print "The following build artifacts are not specified:" + print " " + ", ".join(missed) + sys.exit(1) + + if options.image_name: + BB_VARS.default_image = options.image_name + else: + options.build_check = False + + if options.vars_dir: + BB_VARS.vars_dir = options.vars_dir + + if options.build_check: + print "Checking basic build environment..." + if not engine.verify_build_env(): + print "Couldn't verify build environment, exiting\n" + sys.exit(1) + else: + print "Done.\n" + + bootimg_dir = "" + + if options.image_name: + if options.build_rootfs: + argv = ["bitbake", options.image_name] + if options.debug: + argv.append("--debug") + + print "Building rootfs...\n" + if bitbake_main(BitBakeConfigParameters(argv), + cookerdata.CookerConfiguration()): + sys.exit(1) + + rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", options.image_name) + kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE", options.image_name) + native_sysroot = get_bitbake_var("STAGING_DIR_NATIVE", + options.image_name) + else: + if options.build_rootfs: + print "Image name is not specified, exiting. (Use -e/--image-name to specify it)\n" + sys.exit(1) + + wks_file = args[0] + + if not wks_file.endswith(".wks"): + wks_file = engine.find_canned_image(scripts_path, wks_file) + if not wks_file: + print "No image named %s found, exiting. (Use 'wic list images' "\ + "to list available images, or specify a fully-qualified OE "\ + "kickstart (.wks) filename)\n" % args[0] + sys.exit(1) + + image_output_dir = "" + if options.outdir: + image_output_dir = options.outdir + + if not options.image_name: + rootfs_dir = '' + if 'ROOTFS_DIR' in options.rootfs_dir: + rootfs_dir = options.rootfs_dir['ROOTFS_DIR'] + bootimg_dir = options.bootimg_dir + kernel_dir = options.kernel_dir + native_sysroot = options.native_sysroot + if rootfs_dir and not os.path.isdir(rootfs_dir): + print "--roofs-dir (-r) not found, exiting\n" + sys.exit(1) + if not os.path.isdir(bootimg_dir): + print "--bootimg-dir (-b) not found, exiting\n" + sys.exit(1) + if not os.path.isdir(kernel_dir): + print "--kernel-dir (-k) not found, exiting\n" + sys.exit(1) + if not os.path.isdir(native_sysroot): + print "--native-sysroot (-n) not found, exiting\n" + sys.exit(1) + else: + not_found = not_found_dir = "" + if not os.path.isdir(rootfs_dir): + (not_found, not_found_dir) = ("rootfs-dir", rootfs_dir) + elif not os.path.isdir(kernel_dir): + (not_found, not_found_dir) = ("kernel-dir", kernel_dir) + elif not os.path.isdir(native_sysroot): + (not_found, not_found_dir) = ("native-sysroot", native_sysroot) + if not_found: + if not not_found_dir: + not_found_dir = "Completely missing artifact - wrong image (.wks) used?" + print "Build artifacts not found, exiting." + print " (Please check that the build artifacts for the machine" + print " selected in local.conf actually exist and that they" + print " are the correct artifacts for the image (.wks file)).\n" + print "The artifact that couldn't be found was %s:\n %s" % \ + (not_found, not_found_dir) + sys.exit(1) + + krootfs_dir = options.rootfs_dir + if krootfs_dir is None: + krootfs_dir = {} + krootfs_dir['ROOTFS_DIR'] = rootfs_dir + + rootfs_dir = rootfs_dir_to_args(krootfs_dir) + + print "Creating image(s)...\n" + engine.wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, + native_sysroot, scripts_path, image_output_dir, + options.compressor, options.debug) + + +def wic_list_subcommand(args, usage_str): + """ + Command-line handling for listing available images. + The real work is done by image.engine.wic_list() + """ + parser = optparse.OptionParser(usage=usage_str) + args = parser.parse_args(args)[1] + + if not engine.wic_list(args, scripts_path): + logging.error("Bad list arguments, exiting\n") + parser.print_help() + sys.exit(1) + + +def wic_help_topic_subcommand(args, usage_str): + """ + Command-line handling for help-only 'subcommands'. This is + essentially a dummy command that doesn nothing but allow users to + use the existing subcommand infrastructure to display help on a + particular topic not attached to any particular subcommand. + """ + pass + + +wic_help_topic_usage = """ +""" + +subcommands = { + "create": [wic_create_subcommand, + hlp.wic_create_usage, + hlp.wic_create_help], + "list": [wic_list_subcommand, + hlp.wic_list_usage, + hlp.wic_list_help], + "plugins": [wic_help_topic_subcommand, + wic_help_topic_usage, + hlp.get_wic_plugins_help], + "overview": [wic_help_topic_subcommand, + wic_help_topic_usage, + hlp.wic_overview_help], + "kickstart": [wic_help_topic_subcommand, + wic_help_topic_usage, + hlp.wic_kickstart_help], +} + + +def start_logging(loglevel): + logging.basicConfig(filname='wic.log', filemode='w', level=loglevel) + + +def main(argv): + parser = optparse.OptionParser(version="wic version %s" % __version__, + usage=hlp.wic_usage) + + parser.disable_interspersed_args() + + args = parser.parse_args(argv)[1] + + if len(args): + if args[0] == "help": + if len(args) == 1: + parser.print_help() + sys.exit(1) + + return hlp.invoke_subcommand(args, parser, hlp.wic_help_usage, subcommands) + + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[1:])) + except WicError as err: + print >> sys.stderr, "ERROR:", err + sys.exit(1) + diff --git a/scripts/wipe-sysroot b/scripts/wipe-sysroot new file mode 100755 index 0000000..9e067e8 --- /dev/null +++ b/scripts/wipe-sysroot @@ -0,0 +1,54 @@ +#! /bin/sh + +# Wipe out all of the sysroots and all of the stamps that populated it. +# Author: Ross Burton <ross.burton@intel.com> +# +# Copyright (c) 2012 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +set -e + +if [ $# -gt 0 ]; then + echo "Wipe all sysroots and sysroot-related stamps for the current build directory." >&2 + echo "Usage: $0" >&2 + exit 1 +fi + +ENVS=`mktemp --suffix -wipe-sysroot-envs` +bitbake -p -e > $ENVS + +eval `grep -F SSTATE_MANIFESTS= $ENVS` +eval `grep -F STAGING_DIR= $ENVS` +eval `grep -F STAMPS_DIR= $ENVS` +rm -f $ENVS + +if [ -z "$SSTATE_MANIFESTS" -o -z "$STAGING_DIR" -o -z "$STAMPS_DIR" ]; then + echo "Could not determine SSTATE_MANIFESTS/STAGING_DIR/STAMPS_DIR from bitbake, check above for errors" + exit 1 +fi + +echo "Deleting the sysroots in $STAGING_DIR, and selected stamps in $SSTATE_MANIFESTS and $STAMPS_DIR." + +# The sysroots themselves +rm -rf $STAGING_DIR ${STAGING_DIR}-uninative + +# The stamps that said the sysroot was populated +rm -rf $STAMPS_DIR/*/*/*.do_populate_sysroot.* +rm -rf $STAMPS_DIR/*/*/*.do_populate_sysroot_setscene.* +rm -rf $STAMPS_DIR/*/*/*.do_packagedata.* +rm -rf $STAMPS_DIR/*/*/*.do_packagedata_setscene.* + +# The sstate manifests +rm -rf $SSTATE_MANIFESTS/manifest-*.populate-sysroot diff --git a/scripts/yocto-bsp b/scripts/yocto-bsp new file mode 100755 index 0000000..82a050e --- /dev/null +++ b/scripts/yocto-bsp @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# 'yocto-bsp' is the Yocto BSP Tool that helps users create a new +# Yocto BSP. Invoking it without any arguments will display help +# screens for the 'yocto-bsp' command and list the available +# 'yocto-bsp' subcommands. Invoking a subcommand without any +# arguments will likewise display help screens for the specified +# subcommand. Please use that interface for detailed help. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] intel.com> +# + +__version__ = "0.1.0" + +import os +import sys +import optparse +import logging + +scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] + +from bsp.help import * +from bsp.engine import * + + +def yocto_bsp_create_subcommand(args, usage_str): + """ + Command-line handling for BSP creation. The real work is done by + bsp.engine.yocto_bsp_create() + """ + parser = optparse.OptionParser(usage = usage_str) + + parser.add_option("-o", "--outdir", dest = "outdir", action = "store", + help = "name of BSP dir to create") + parser.add_option("-i", "--infile", dest = "properties_file", action = "store", + help = "name of file containing the values for BSP properties as a JSON file") + parser.add_option("-c", "--codedump", dest = "codedump", action = "store_true", + default = False, help = "dump the generated code to bspgen.out") + parser.add_option("-s", "--skip-git-check", dest = "git_check", action = "store_false", + default = True, help = "skip the git connectivity check") + (options, args) = parser.parse_args(args) + + if len(args) != 2: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + machine = args[0] + karch = args[1] + + if options.outdir: + bsp_output_dir = options.outdir + else: + bsp_output_dir = "meta-" + machine + + if options.git_check and not options.properties_file: + print "Checking basic git connectivity..." + if not verify_git_repo(GIT_CHECK_URI): + print "Couldn't verify git connectivity, exiting\n" + print "Details: couldn't access %s" % GIT_CHECK_URI + print " (this most likely indicates a network connectivity problem or" + print " a misconfigured git intallation)" + sys.exit(1) + else: + print "Done.\n" + + yocto_bsp_create(machine, karch, scripts_path, bsp_output_dir, options.codedump, options.properties_file) + + +def yocto_bsp_list_subcommand(args, usage_str): + """ + Command-line handling for listing available BSP properties and + values. The real work is done by bsp.engine.yocto_bsp_list() + """ + parser = optparse.OptionParser(usage = usage_str) + + parser.add_option("-o", "--outfile", action = "store", dest = "properties_file", + help = "dump the possible values for BSP properties to a JSON file") + + (options, args) = parser.parse_args(args) + + if not yocto_bsp_list(args, scripts_path, options.properties_file): + logging.error("Bad list arguments, exiting\n") + parser.print_help() + sys.exit(1) + + +subcommands = { + "create": [yocto_bsp_create_subcommand, + yocto_bsp_create_usage, + yocto_bsp_create_help], + "list": [yocto_bsp_list_subcommand, + yocto_bsp_list_usage, + yocto_bsp_list_help], +} + + +def start_logging(loglevel): + logging.basicConfig(filname = 'yocto-bsp.log', filemode = 'w', level=loglevel) + + +def main(): + parser = optparse.OptionParser(version = "yocto-bsp version %s" % __version__, + usage = yocto_bsp_usage) + + parser.disable_interspersed_args() + parser.add_option("-D", "--debug", dest = "debug", action = "store_true", + default = False, help = "output debug information") + + (options, args) = parser.parse_args() + + loglevel = logging.INFO + if options.debug: + loglevel = logging.DEBUG + start_logging(loglevel) + + if len(args): + if args[0] == "help": + if len(args) == 1: + parser.print_help() + sys.exit() + + invoke_subcommand(args, parser, yocto_bsp_help_usage, subcommands) + + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) + diff --git a/scripts/yocto-kernel b/scripts/yocto-kernel new file mode 100755 index 0000000..daaad07 --- /dev/null +++ b/scripts/yocto-kernel @@ -0,0 +1,399 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# 'yocto-kernel' is the Yocto BSP Tool that helps users manage kernel +# config options and patches for a Yocto BSP. Invoking it without any +# arguments will display help screens for the 'yocto-kernel' command +# and list the available 'yocto-kernel' subcommands. Invoking a +# subcommand without any arguments will likewise display help screens +# for the specified subcommand. Please use that interface for +# detailed help. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] intel.com> +# + +__version__ = "0.1.0" + +import os +import sys +import optparse +import logging + +scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] + +from bsp.help import * +from bsp.kernel import * + + +def yocto_kernel_config_list_subcommand(args, usage_str): + """ + Command-line handling for listing BSP config options. The + real work is done by bsp.kernel.yocto_kernel_config_list(). + """ + logging.debug("yocto_kernel_config_list_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) != 1: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + yocto_kernel_config_list(scripts_path, args[0]) + + +def yocto_kernel_config_add_subcommand(args, usage_str): + """ + Command-line handling for adding BSP config items. The real work + is done by bsp.kernel.yocto_kernel_config_add(). + """ + logging.debug("yocto_kernel_config_add_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) < 2: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + machine = args.pop(0) + yocto_kernel_config_add(scripts_path, machine, args) + + +def yocto_kernel_config_rm_subcommand(args, usage_str): + """ + Command-line handling for removing BSP config items. The real + work is done by bsp.kernel.yocto_kernel_config_rm(). + """ + logging.debug("yocto_kernel_config_rm_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) != 1: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + yocto_kernel_config_rm(scripts_path, args[0]) + + +def yocto_kernel_patch_list_subcommand(args, usage_str): + """ + Command-line handling for listing BSP (SRC_URI patches. The real + work is done by bsp.kernel.yocto_kernel_patch_list(). + """ + logging.debug("yocto_kernel_patch_list_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) != 1: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + yocto_kernel_patch_list(scripts_path, args[0]) + + +def yocto_kernel_patch_add_subcommand(args, usage_str): + """ + Command-line handling for adding BSP patches. The real work is + done by bsp.kernel.yocto_kernel_patch_add(). + """ + logging.debug("yocto_kernel_patch_add_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) < 2: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + machine = args.pop(0) + yocto_kernel_patch_add(scripts_path, machine, args) + + +def yocto_kernel_patch_rm_subcommand(args, usage_str): + """ + Command-line handling for removing BSP patches. The real work is + done by bsp.kernel.yocto_kernel_patch_rm(). + """ + logging.debug("yocto_kernel_patch_rm_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) != 1: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + yocto_kernel_patch_rm(scripts_path, args[0]) + + +def yocto_kernel_feature_list_subcommand(args, usage_str): + """ + Command-line handling for listing the BSP features that are being + used by the BSP. The real work is done by + bsp.kernel.yocto_kernel_feature_list(). + """ + logging.debug("yocto_kernel_feature_list_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) != 1: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + yocto_kernel_feature_list(scripts_path, args[0]) + + +def yocto_kernel_feature_add_subcommand(args, usage_str): + """ + Command-line handling for adding the use of kernel features to a + BSP. The real work is done by bsp.kernel.yocto_kernel_feature_add(). + """ + logging.debug("yocto_kernel_feature_add_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) < 2: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + machine = args.pop(0) + yocto_kernel_feature_add(scripts_path, machine, args) + + +def yocto_kernel_feature_rm_subcommand(args, usage_str): + """ + Command-line handling for removing the use of kernel features from + a BSP. The real work is done by bsp.kernel.yocto_kernel_feature_rm(). + """ + logging.debug("yocto_kernel_feature_rm_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) != 1: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + yocto_kernel_feature_rm(scripts_path, args[0]) + + +def yocto_kernel_available_features_list_subcommand(args, usage_str): + """ + Command-line handling for listing all the kernel features + available for use in a BSP. This includes the features present in + the meta branch(es) of the pointed-to repo(s) as well as the local + features added in recipe-space to the current BSP as well. The + real work is done by bsp.kernel.yocto_kernel_available_features_list(). + """ + logging.debug("yocto_kernel_feature_available_features_list_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) != 1: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + yocto_kernel_available_features_list(scripts_path, args[0]) + + +def yocto_kernel_feature_describe_subcommand(args, usage_str): + """ + Command-line handling for listing the description of a specific + kernel feature available for use in a BSP. This includes the + features present in the meta branch(es) of the pointed-to repo(s) + as well as the local features added in recipe-space to the current + BSP as well. The real work is done by + bsp.kernel.yocto_kernel_feature_describe(). + """ + logging.debug("yocto_kernel_feature_describe_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) != 2: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + yocto_kernel_feature_describe(scripts_path, args[0], args[1]) + + +def yocto_kernel_feature_create_subcommand(args, usage_str): + """ + Command-line handling for creating a recipe-space kernel feature + in a BSP. The real work is done by + bsp.kernel.yocto_kernel_feature_create(). + """ + logging.debug("yocto_kernel_feature_create_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) < 4: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + machine = args.pop(0) + yocto_kernel_feature_create(scripts_path, machine, args) + + +def yocto_kernel_feature_destroy_subcommand(args, usage_str): + """ + Command-line handling for removing a recipe-space kernel feature + from a BSP. The real work is done by + bsp.kernel.yocto_kernel_feature_destroy(). + """ + logging.debug("yocto_kernel_feature_destroy_subcommand") + + parser = optparse.OptionParser(usage = usage_str) + + (options, args) = parser.parse_args(args) + + if len(args) != 2: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + yocto_kernel_feature_destroy(scripts_path, args[0], args[1]) + + +subcommands = { + "config-list": [yocto_kernel_config_list_subcommand, + yocto_kernel_config_list_usage, + yocto_kernel_config_list_help], + "config-add": [yocto_kernel_config_add_subcommand, + yocto_kernel_config_add_usage, + yocto_kernel_config_add_help], + "config-rm": [yocto_kernel_config_rm_subcommand, + yocto_kernel_config_rm_usage, + yocto_kernel_config_rm_help], + "patch-list": [yocto_kernel_patch_list_subcommand, + yocto_kernel_patch_list_usage, + yocto_kernel_patch_list_help], + "patch-add": [yocto_kernel_patch_add_subcommand, + yocto_kernel_patch_add_usage, + yocto_kernel_patch_add_help], + "patch-rm": [yocto_kernel_patch_rm_subcommand, + yocto_kernel_patch_rm_usage, + yocto_kernel_patch_rm_help], + "feature-list": [yocto_kernel_feature_list_subcommand, + yocto_kernel_feature_list_usage, + yocto_kernel_feature_list_help], + "feature-add": [yocto_kernel_feature_add_subcommand, + yocto_kernel_feature_add_usage, + yocto_kernel_feature_add_help], + "feature-rm": [yocto_kernel_feature_rm_subcommand, + yocto_kernel_feature_rm_usage, + yocto_kernel_feature_rm_help], + "features-list": [yocto_kernel_available_features_list_subcommand, + yocto_kernel_available_features_list_usage, + yocto_kernel_available_features_list_help], + "feature-describe": [yocto_kernel_feature_describe_subcommand, + yocto_kernel_feature_describe_usage, + yocto_kernel_feature_describe_help], + "feature-create": [yocto_kernel_feature_create_subcommand, + yocto_kernel_feature_create_usage, + yocto_kernel_feature_create_help], + "feature-destroy": [yocto_kernel_feature_destroy_subcommand, + yocto_kernel_feature_destroy_usage, + yocto_kernel_feature_destroy_help], +} + + +def start_logging(loglevel): + logging.basicConfig(filname = 'yocto-kernel.log', filemode = 'w', level=loglevel) + + +def main(): + parser = optparse.OptionParser(version = "yocto-kernel version %s" % __version__, + usage = yocto_kernel_usage) + + parser.disable_interspersed_args() + parser.add_option("-D", "--debug", dest = "debug", action = "store_true", + default = False, help = "output debug information") + + (options, args) = parser.parse_args() + + loglevel = logging.INFO + if options.debug: + loglevel = logging.DEBUG + start_logging(loglevel) + + if len(args): + if args[0] == "help": + if len(args) == 1: + parser.print_help() + sys.exit(1) + sc = 1 + else: + sc = 0 + + if args[sc] == "config" or args[sc] == "patch" or \ + args[sc] == "feature" or args[sc] == "features": + if len(args) < 2 + sc: + parser.print_help() + sys.exit(1) + args[sc] += "-" + args[sc + 1] + args.pop(sc + 1) + + invoke_subcommand(args, parser, yocto_kernel_help_usage, subcommands) + + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/scripts/yocto-layer b/scripts/yocto-layer new file mode 100755 index 0000000..356972e --- /dev/null +++ b/scripts/yocto-layer @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# 'yocto-layer' is the Yocto Tool that helps users create a new Yocto +# layer. Invoking it without any arguments will display help screens +# for the 'yocto-layer' command and list the available 'yocto-layer' +# subcommands. Invoking a subcommand without any arguments will +# likewise display help screens for the specified subcommand. Please +# use that interface for detailed help. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] intel.com> +# + +__version__ = "0.1.0" + +import os +import sys +import optparse +import logging + +scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] + +from bsp.help import * +from bsp.engine import * + + +def yocto_layer_create_subcommand(args, usage_str): + """ + Command-line handling for layer creation. The real work is done by + bsp.engine.yocto_layer_create() + """ + parser = optparse.OptionParser(usage = usage_str) + + parser.add_option("-o", "--outdir", dest = "outdir", action = "store", + help = "name of layer dir to create") + parser.add_option("-i", "--infile", dest = "properties_file", action = "store", + help = "name of file containing the values for layer input properties as a JSON file") + parser.add_option("-c", "--codedump", dest = "codedump", action = "store_true", + default = False, help = "dump the generated code to layergen.out") + (options, args) = parser.parse_args(args) + + if len(args) < 1 or len(args) > 2: + logging.error("Wrong number of arguments, exiting\n") + parser.print_help() + sys.exit(1) + + layer_name = args[0] + properties = "" + + if len(args) == 2: + layer_priority = args[1] + properties = '{"layer_priority":"' + layer_priority + '"}' + + if options.outdir: + layer_output_dir = options.outdir + else: + prefix="meta-" + if not layer_name.startswith(prefix): + layer_output_dir="%s%s"%(prefix,layer_name) + else: + layer_output_dir=layer_name + + yocto_layer_create(layer_name, scripts_path, layer_output_dir, options.codedump, options.properties_file, properties) + + +def yocto_layer_list_subcommand(args, usage_str): + """ + Command-line handling for listing available layer properties and + values. The real work is done by bsp.engine.yocto_layer_list() + """ + parser = optparse.OptionParser(usage = usage_str) + + parser.add_option("-o", "--outfile", action = "store", dest = "properties_file", + help = "dump the possible values for layer properties to a JSON file") + + (options, args) = parser.parse_args(args) + + if not yocto_layer_list(args, scripts_path, options.properties_file): + logging.error("Bad list arguments, exiting\n") + parser.print_help() + sys.exit(1) + + +subcommands = { + "create": [yocto_layer_create_subcommand, + yocto_layer_create_usage, + yocto_layer_create_help], + "list": [yocto_layer_list_subcommand, + yocto_layer_list_usage, + yocto_layer_list_help], +} + + +def start_logging(loglevel): + logging.basicConfig(filname = 'yocto-layer.log', filemode = 'w', level=loglevel) + + +def main(): + parser = optparse.OptionParser(version = "yocto-layer version %s" % __version__, + usage = yocto_layer_usage) + + parser.disable_interspersed_args() + parser.add_option("-D", "--debug", dest = "debug", action = "store_true", + default = False, help = "output debug information") + + (options, args) = parser.parse_args() + + loglevel = logging.INFO + if options.debug: + loglevel = logging.DEBUG + start_logging(loglevel) + + if len(args): + if args[0] == "help": + if len(args) == 1: + parser.print_help() + sys.exit(1) + + invoke_subcommand(args, parser, yocto_layer_help_usage, subcommands) + + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) + diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..c0b6230 --- /dev/null +++ b/setup.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +set -e +BUILDCONF=conf/local.conf +BBLAYERS=conf/bblayers.conf + +if [ "$1" != "--update" ]; then + echo "" + echo "Setting up git hooks..." + ln -s ../../scripts/git-hooks/post-merge .git/hooks/post-merge || true + ln -s ../../scripts/git-hooks/post-checkout .git/hooks/post-checkout || true + ln -s ../../scripts/git-hooks/pre-commit .git/hooks/pre-commit || true + ln -s ../../scripts/git-hooks/pre-push .git/hooks/pre-push || true +fi + +echo "" +echo "Updating git submodules..." +git submodule update --init --recursive + +# Apply patches +for f in $(ls patches) ; do + if patch --dry-run -Np1 < patches/$f ; then + patch -Np1 < patches/$f + fi +done + +# Remove yocto-linux bbappend +rm -f layers/meta-virtualization/recipes-kernel/linux/linux-yocto_*.bbappend + +if [ "$1" != "--update" ]; then + if [ ! -d build/conf ]; then + mkdir -p build/conf + fi + + echo "" + response=y + test -f ${BUILDCONF} && read -p "Overwrite ${BUILDCONF} with defaults? (y/N) " response + if [[ $response =~ ^y$ || $response =~ ^yes$ ]]; then + echo "Creating default bitbake configuration..." + cp layers/meta-mlinux/contrib/local.conf conf/ + else + echo "Leaving existing ${BUILDCONF} alone." + fi + + root_pwd_hash=$(egrep '^MTADM_PASSWORD_HASH[[:space:]]*=' ${BUILDCONF} || true) + if ((${#root_pwd_hash} == 0)) ; then + if [[ "$MTADM_PASSWORD" ]] ; then + pass=$MTADM_PASSWORD + else + pass=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 2>/dev/null | head -c${1:-8};echo) + fi + salt="$(openssl rand -base64 128 2>/dev/null)" + hash="$(openssl passwd -1 -salt "$salt" "$pass")" + echo "MTADM_PASSWORD = \"$pass\"" >password.txt + echo "HASH = \"$hash\"" >>password.txt + sed -ri "/MTADM_PASSWORD_HASH[[:space:]]=/d" ${BUILDCONF} || true + echo "MTADM_PASSWORD_HASH = \"$hash\"" >>${BUILDCONF} + sed -ri "/MTADM_PASSWORD[[:space:]]=/d" ${BUILDCONF} || true + echo "MTADM_PASSWORD = \"$pass\"" >>${BUILDCONF} + fi + + echo "" + response=y + test -f $BBLAYERS && read -p "Overwrite $BBLAYERS with defaults? (y/N) " response + if [[ $response =~ ^y$ || $response =~ ^yes$ ]]; then + echo "Copying default bblayers..." + OEROOT=$(pwd) + sed -e "s?__OEROOT__?${OEROOT}?" conf/bblayers.conf.mlinux >$BBLAYERS + else + echo "Leaving existing $BBLAYERS alone." + fi + + echo "" + echo "Creating user-layer..." + mkdir -p layers/user-layer/conf + mkdir -p layers/user-layer/recipes-core + + echo "" + response=y + test -f layers/user-layer/conf/layer.conf && read -p "Overwrite layers/user-layer/conf/layer.conf with defaults? (y/N) " response + if [[ $response =~ ^y$ || $response =~ ^yes$ ]]; then + echo "Creating default user-layer configuration..." + cp layers/meta-mlinux/contrib/user-layer.conf layers/user-layer/conf/layer.conf + else + echo "Leaving existing layers/user-layer/conf/layer.conf alone." + fi + + echo "" + echo "Creating directory structure..." + mkdir -p build + mkdir -p downloads +fi + +echo "" +echo "Setup Done." +echo "" +echo "oe-init-build-env will set the current directory to build" +echo "One must be within the build tree to run bitbake." +echo "" +echo "To build mlinux-base-image:" +echo " source oe-init-build-env" +echo " machine=mtcdt bitbake mlinux-base-image" +echo " or (requires Redpine sources)" +echo " machine=mtcdt bitbake mlinux-commisioning-image" +echo "" +echo "To build mlinux-mtcap-image:" +echo " source oe-init-build-env" +echo " MACHINE=mtcap bitbake mlinux-mtcap-image" +echo " or" +echo " MACHINE=mtcap bitbake mlinux-mtcap-commisioning-image" + |