diff options
Diffstat (limited to 'scripts/lib/devtool')
| -rw-r--r-- | scripts/lib/devtool/__init__.py | 236 | ||||
| -rw-r--r-- | scripts/lib/devtool/build.py | 86 | ||||
| -rw-r--r-- | scripts/lib/devtool/build_image.py | 174 | ||||
| -rw-r--r-- | scripts/lib/devtool/build_sdk.py | 65 | ||||
| -rw-r--r-- | scripts/lib/devtool/deploy.py | 323 | ||||
| -rw-r--r-- | scripts/lib/devtool/package.py | 60 | ||||
| -rw-r--r-- | scripts/lib/devtool/runqemu.py | 74 | ||||
| -rw-r--r-- | scripts/lib/devtool/sdk.py | 336 | ||||
| -rw-r--r-- | scripts/lib/devtool/search.py | 88 | ||||
| -rw-r--r-- | scripts/lib/devtool/standard.py | 2150 | ||||
| -rw-r--r-- | scripts/lib/devtool/upgrade.py | 409 | ||||
| -rw-r--r-- | scripts/lib/devtool/utilcmds.py | 233 |
12 files changed, 3667 insertions, 567 deletions
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py index 4c0d5397dc..d646b0cf63 100644 --- a/scripts/lib/devtool/__init__.py +++ b/scripts/lib/devtool/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Development tool - utility functions for plugins # @@ -16,16 +16,26 @@ # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - +"""Devtool plugins module""" import os import sys import subprocess import logging +import re +import codecs logger = logging.getLogger('devtool') +class DevtoolError(Exception): + """Exception for handling devtool errors""" + def __init__(self, message, exitcode=1): + super(DevtoolError, self).__init__(message) + self.exitcode = exitcode + + def exec_build_env_command(init_path, builddir, cmd, watch=False, **options): + """Run a program in bitbake build context""" import bb if not 'cwd' in options: options["cwd"] = builddir @@ -49,17 +59,19 @@ def exec_build_env_command(init_path, builddir, cmd, watch=False, **options): return bb.process.run('%s%s' % (init_prefix, cmd), **options) def exec_watch(cmd, **options): + """Run program with stdout shown on sys.stdout""" import bb - if isinstance(cmd, basestring) and not "shell" in options: + if isinstance(cmd, str) and not "shell" in options: options["shell"] = True process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options ) + reader = codecs.getreader('utf-8')(process.stdout) buf = '' while True: - out = process.stdout.read(1) + out = reader.read(1, 1) if out: sys.stdout.write(out) sys.stdout.flush() @@ -72,17 +84,211 @@ def exec_watch(cmd, **options): return buf, None -def setup_tinfoil(): +def exec_fakeroot(d, cmd, **kwargs): + """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions""" + # Grab the command and check it actually exists + fakerootcmd = d.getVar('FAKEROOTCMD') + if not os.path.exists(fakerootcmd): + logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built') + return 2 + # Set up the appropriate environment + newenv = dict(os.environ) + fakerootenv = d.getVar('FAKEROOTENV') + for varvalue in fakerootenv.split(): + if '=' in varvalue: + splitval = varvalue.split('=', 1) + newenv[splitval[0]] = splitval[1] + return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs) + +def setup_tinfoil(config_only=False, basepath=None, tracking=False): + """Initialize tinfoil api from bitbake""" import scriptpath - bitbakepath = scriptpath.add_bitbake_lib_path() - if not bitbakepath: - logger.error("Unable to find bitbake by searching parent directory of this script or PATH") - sys.exit(1) - - import bb.tinfoil - import logging - tinfoil = bb.tinfoil.Tinfoil() - tinfoil.prepare(False) - tinfoil.logger.setLevel(logging.WARNING) + orig_cwd = os.path.abspath(os.curdir) + try: + if basepath: + os.chdir(basepath) + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + sys.exit(1) + + import bb.tinfoil + tinfoil = bb.tinfoil.Tinfoil(tracking=tracking) + try: + tinfoil.prepare(config_only) + tinfoil.logger.setLevel(logger.getEffectiveLevel()) + except bb.tinfoil.TinfoilUIException: + tinfoil.shutdown() + raise DevtoolError('Failed to start bitbake environment') + except: + tinfoil.shutdown() + raise + finally: + os.chdir(orig_cwd) return tinfoil +def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True): + """Parse the specified recipe""" + try: + recipefile = tinfoil.get_recipe_file(pn) + except bb.providers.NoProvider as e: + logger.error(str(e)) + return None + if appends: + append_files = tinfoil.get_file_appends(recipefile) + if filter_workspace: + # Filter out appends from the workspace + append_files = [path for path in append_files if + not path.startswith(config.workspace_path)] + else: + append_files = None + try: + rd = tinfoil.parse_recipe_file(recipefile, appends, append_files) + except Exception as e: + logger.error(str(e)) + return None + return rd + +def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False): + """ + Check that a recipe is in the workspace and (optionally) that source + is present. + """ + + workspacepn = pn + + for recipe, value in workspace.items(): + if recipe == pn: + break + if bbclassextend: + recipefile = value['recipefile'] + if recipefile: + targets = get_bbclassextend_targets(recipefile, recipe) + if pn in targets: + workspacepn = recipe + break + else: + raise DevtoolError("No recipe named '%s' in your workspace" % pn) + + if checksrc: + srctree = workspace[workspacepn]['srctree'] + if not os.path.exists(srctree): + raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn)) + if not os.listdir(srctree): + raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn)) + + return workspacepn + +def use_external_build(same_dir, no_same_dir, d): + """ + Determine if we should use B!=S (separate build and source directories) or not + """ + b_is_s = True + if no_same_dir: + logger.info('Using separate build directory since --no-same-dir specified') + b_is_s = False + elif same_dir: + logger.info('Using source tree as build directory since --same-dir specified') + elif bb.data.inherits_class('autotools-brokensep', d): + logger.info('Using source tree as build directory since recipe inherits autotools-brokensep') + elif d.getVar('B') == os.path.abspath(d.getVar('S')): + logger.info('Using source tree as build directory since that would be the default for this recipe') + else: + b_is_s = False + return b_is_s + +def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None): + """ + Set up the git repository for the source tree + """ + import bb.process + import oe.patch + if not os.path.exists(os.path.join(repodir, '.git')): + bb.process.run('git init', cwd=repodir) + bb.process.run('git add .', cwd=repodir) + commit_cmd = ['git'] + oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d) + commit_cmd += ['commit', '-q'] + stdout, _ = bb.process.run('git status --porcelain', cwd=repodir) + if not stdout: + commit_cmd.append('--allow-empty') + commitmsg = "Initial empty commit with no upstream sources" + elif version: + commitmsg = "Initial commit from upstream at version %s" % version + else: + commitmsg = "Initial commit from upstream" + commit_cmd += ['-m', commitmsg] + bb.process.run(commit_cmd, cwd=repodir) + + bb.process.run('git checkout -b %s' % devbranch, cwd=repodir) + bb.process.run('git tag -f %s' % basetag, cwd=repodir) + +def recipe_to_append(recipefile, config, wildcard=False): + """ + Convert a recipe file to a bbappend file path within the workspace. + NOTE: if the bbappend already exists, you should be using + workspace[args.recipename]['bbappend'] instead of calling this + function. + """ + appendname = os.path.splitext(os.path.basename(recipefile))[0] + if wildcard: + appendname = re.sub(r'_.*', '_%', appendname) + appendpath = os.path.join(config.workspace_path, 'appends') + appendfile = os.path.join(appendpath, appendname + '.bbappend') + return appendfile + +def get_bbclassextend_targets(recipefile, pn): + """ + Cheap function to get BBCLASSEXTEND and then convert that to the + list of targets that would result. + """ + import bb.utils + + values = {} + def get_bbclassextend_varfunc(varname, origvalue, op, newlines): + values[varname] = origvalue + return origvalue, None, 0, True + with open(recipefile, 'r') as f: + bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc) + + targets = [] + bbclassextend = values.get('BBCLASSEXTEND', '').split() + if bbclassextend: + for variant in bbclassextend: + if variant == 'nativesdk': + targets.append('%s-%s' % (variant, pn)) + elif variant in ['native', 'cross', 'crosssdk']: + targets.append('%s-%s' % (pn, variant)) + return targets + +def ensure_npm(config, basepath, fixed_setup=False, check_exists=True): + """ + Ensure that npm is available and either build it or show a + reasonable error message + """ + if check_exists: + tinfoil = setup_tinfoil(config_only=False, basepath=basepath) + try: + rd = tinfoil.parse_recipe('nodejs-native') + nativepath = rd.getVar('STAGING_BINDIR_NATIVE') + finally: + tinfoil.shutdown() + npmpath = os.path.join(nativepath, 'npm') + build_npm = not os.path.exists(npmpath) + else: + build_npm = True + + if build_npm: + logger.info('Building nodejs-native') + try: + exec_build_env_command(config.init_path, basepath, + 'bitbake -q nodejs-native -c addto_recipe_sysroot', watch=True) + except bb.process.ExecutionError as e: + if "Nothing PROVIDES 'nodejs-native'" in e.stdout: + if fixed_setup: + msg = 'nodejs-native is required for npm but is not available within this SDK' + else: + msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs' + raise DevtoolError(msg) + else: + raise diff --git a/scripts/lib/devtool/build.py b/scripts/lib/devtool/build.py new file mode 100644 index 0000000000..252379e9b2 --- /dev/null +++ b/scripts/lib/devtool/build.py @@ -0,0 +1,86 @@ +# Development tool - build command plugin +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool build plugin""" + +import os +import bb +import logging +import argparse +import tempfile +from devtool import exec_build_env_command, check_workspace_recipe, DevtoolError + +logger = logging.getLogger('devtool') + + +def _set_file_values(fn, values): + remaining = list(values.keys()) + + def varfunc(varname, origvalue, op, newlines): + newvalue = values.get(varname, origvalue) + remaining.remove(varname) + return (newvalue, '=', 0, True) + + with open(fn, 'r') as f: + (updated, newlines) = bb.utils.edit_metadata(f, values, varfunc) + + for item in remaining: + updated = True + newlines.append('%s = "%s"' % (item, values[item])) + + if updated: + with open(fn, 'w') as f: + f.writelines(newlines) + return updated + +def _get_build_tasks(config): + tasks = config.get('Build', 'build_task', 'populate_sysroot,packagedata').split(',') + return ['do_%s' % task.strip() for task in tasks] + +def build(args, config, basepath, workspace): + """Entry point for the devtool 'build' subcommand""" + workspacepn = check_workspace_recipe(workspace, args.recipename, bbclassextend=True) + + build_tasks = _get_build_tasks(config) + + bbappend = workspace[workspacepn]['bbappend'] + if args.disable_parallel_make: + logger.info("Disabling 'make' parallelism") + _set_file_values(bbappend, {'PARALLEL_MAKE': ''}) + try: + bbargs = [] + for task in build_tasks: + if args.recipename.endswith('-native') and 'package' in task: + continue + bbargs.append('%s:%s' % (args.recipename, task)) + exec_build_env_command(config.init_path, basepath, 'bitbake %s' % ' '.join(bbargs), watch=True) + except bb.process.ExecutionError as e: + # We've already seen the output since watch=True, so just ensure we return something to the user + return e.exitcode + finally: + if args.disable_parallel_make: + _set_file_values(bbappend, {'PARALLEL_MAKE': None}) + + return 0 + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + parser_build = subparsers.add_parser('build', help='Build a recipe', + description='Builds the specified recipe using bitbake (up to and including %s)' % ', '.join(_get_build_tasks(context.config)), + group='working', order=50) + parser_build.add_argument('recipename', help='Recipe to build') + parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism') + parser_build.set_defaults(func=build) diff --git a/scripts/lib/devtool/build_image.py b/scripts/lib/devtool/build_image.py new file mode 100644 index 0000000000..e5810389be --- /dev/null +++ b/scripts/lib/devtool/build_image.py @@ -0,0 +1,174 @@ +# Development tool - build-image plugin +# +# Copyright (C) 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +"""Devtool plugin containing the build-image subcommand.""" + +import os +import errno +import logging + +from bb.process import ExecutionError +from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError + +logger = logging.getLogger('devtool') + +class TargetNotImageError(Exception): + pass + +def _get_packages(tinfoil, workspace, config): + """Get list of packages from recipes in the workspace.""" + result = [] + for recipe in workspace: + data = parse_recipe(config, tinfoil, recipe, True) + if 'class-target' in data.getVar('OVERRIDES').split(':'): + if recipe in data.getVar('PACKAGES').split(): + result.append(recipe) + else: + logger.warning("Skipping recipe %s as it doesn't produce a " + "package with the same name", recipe) + return result + +def build_image(args, config, basepath, workspace): + """Entry point for the devtool 'build-image' subcommand.""" + + image = args.imagename + auto_image = False + if not image: + sdk_targets = config.get('SDK', 'sdk_targets', '').split() + if sdk_targets: + image = sdk_targets[0] + auto_image = True + if not image: + raise DevtoolError('Unable to determine image to build, please specify one') + + try: + if args.add_packages: + add_packages = args.add_packages.split(',') + else: + add_packages = None + result, outputdir = build_image_task(config, basepath, workspace, image, add_packages) + except TargetNotImageError: + if auto_image: + raise DevtoolError('Unable to determine image to build, please specify one') + else: + raise DevtoolError('Specified recipe %s is not an image recipe' % image) + + if result == 0: + logger.info('Successfully built %s. You can find output files in %s' + % (image, outputdir)) + return result + +def build_image_task(config, basepath, workspace, image, add_packages=None, task=None, extra_append=None): + # remove <image>.bbappend to make sure setup_tinfoil doesn't + # break because of it + target_basename = config.get('SDK', 'target_basename', '') + if target_basename: + appendfile = os.path.join(config.workspace_path, 'appends', + '%s.bbappend' % target_basename) + try: + os.unlink(appendfile) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + + tinfoil = setup_tinfoil(basepath=basepath) + try: + rd = parse_recipe(config, tinfoil, image, True) + if not rd: + # Error already shown + return (1, None) + if not bb.data.inherits_class('image', rd): + raise TargetNotImageError() + + # Get the actual filename used and strip the .bb and full path + target_basename = rd.getVar('FILE') + target_basename = os.path.splitext(os.path.basename(target_basename))[0] + config.set('SDK', 'target_basename', target_basename) + config.write() + + appendfile = os.path.join(config.workspace_path, 'appends', + '%s.bbappend' % target_basename) + + outputdir = None + try: + if workspace or add_packages: + if add_packages: + packages = add_packages + else: + packages = _get_packages(tinfoil, workspace, config) + else: + packages = None + if not task: + if not packages and not add_packages and workspace: + logger.warning('No recipes in workspace, building image %s unmodified', image) + elif not packages: + logger.warning('No packages to add, building image %s unmodified', image) + + if packages or extra_append: + bb.utils.mkdirhier(os.path.dirname(appendfile)) + with open(appendfile, 'w') as afile: + if packages: + # include packages from workspace recipes into the image + afile.write('IMAGE_INSTALL_append = " %s"\n' % ' '.join(packages)) + if not task: + logger.info('Building image %s with the following ' + 'additional packages: %s', image, ' '.join(packages)) + if extra_append: + for line in extra_append: + afile.write('%s\n' % line) + + if task in ['populate_sdk', 'populate_sdk_ext']: + outputdir = rd.getVar('SDK_DEPLOY') + else: + outputdir = rd.getVar('DEPLOY_DIR_IMAGE') + + tmp_tinfoil = tinfoil + tinfoil = None + tmp_tinfoil.shutdown() + + options = '' + if task: + options += '-c %s' % task + + # run bitbake to build image (or specified task) + try: + exec_build_env_command(config.init_path, basepath, + 'bitbake %s %s' % (options, image), watch=True) + except ExecutionError as err: + return (err.exitcode, None) + finally: + if os.path.isfile(appendfile): + os.unlink(appendfile) + finally: + if tinfoil: + tinfoil.shutdown() + return (0, outputdir) + + +def register_commands(subparsers, context): + """Register devtool subcommands from the build-image plugin""" + parser = subparsers.add_parser('build-image', + help='Build image including workspace recipe packages', + description='Builds an image, extending it to include ' + 'packages from recipes in the workspace', + group='testbuild', order=-10) + parser.add_argument('imagename', help='Image recipe to build', nargs='?') + parser.add_argument('-p', '--add-packages', help='Instead of adding packages for the ' + 'entire workspace, specify packages to be added to the image ' + '(separate multiple packages by commas)', + metavar='PACKAGES') + parser.set_defaults(func=build_image) diff --git a/scripts/lib/devtool/build_sdk.py b/scripts/lib/devtool/build_sdk.py new file mode 100644 index 0000000000..b89d65b0cb --- /dev/null +++ b/scripts/lib/devtool/build_sdk.py @@ -0,0 +1,65 @@ +# Development tool - build-sdk command plugin +# +# Copyright (C) 2015-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import subprocess +import logging +import glob +import shutil +import errno +import sys +import tempfile +from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError +from devtool import build_image + +logger = logging.getLogger('devtool') + + +def build_sdk(args, config, basepath, workspace): + """Entry point for the devtool build-sdk command""" + + sdk_targets = config.get('SDK', 'sdk_targets', '').split() + if sdk_targets: + image = sdk_targets[0] + else: + raise DevtoolError('Unable to determine image to build SDK for') + + extra_append = ['SDK_DERIVATIVE = "1"'] + try: + result, outputdir = build_image.build_image_task(config, + basepath, + workspace, + image, + task='populate_sdk_ext', + extra_append=extra_append) + except build_image.TargetNotImageError: + raise DevtoolError('Unable to determine image to build SDK for') + + if result == 0: + logger.info('Successfully built SDK. You can find output files in %s' + % outputdir) + return result + + +def register_commands(subparsers, context): + """Register devtool subcommands""" + if context.fixed_setup: + parser_build_sdk = subparsers.add_parser('build-sdk', + help='Build a derivative SDK of this one', + description='Builds an extensible SDK based upon this one and the items in your workspace', + group='advanced') + parser_build_sdk.set_defaults(func=build_sdk) diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py index f016b23dba..b3730ae833 100644 --- a/scripts/lib/devtool/deploy.py +++ b/scripts/lib/devtool/deploy.py @@ -1,6 +1,6 @@ # Development tool - deploy/undeploy command plugin # -# Copyright (C) 2014-2015 Intel Corporation +# Copyright (C) 2014-2016 Intel Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -14,92 +14,244 @@ # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool plugin containing the deploy subcommands""" import os import subprocess import logging -from devtool import exec_build_env_command +import tempfile +import shutil +import argparse_oe +from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError logger = logging.getLogger('devtool') -def plugin_init(pluginlist): - pass +deploylist_path = '/.devtool' + +def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False): + """ + Prepare a shell script for running on the target to + deploy/undeploy files. We have to be careful what we put in this + script - only commands that are likely to be available on the + target are suitable (the target might be constrained, e.g. using + busybox rather than bash with coreutils). + """ + lines = [] + lines.append('#!/bin/sh') + lines.append('set -e') + if undeployall: + # Yes, I know this is crude - but it does work + lines.append('for entry in %s/*.list; do' % deploylist_path) + lines.append('[ ! -f $entry ] && exit') + lines.append('set `basename $entry | sed "s/.list//"`') + if dryrun: + if not deploy: + lines.append('echo "Previously deployed files for $1:"') + lines.append('manifest="%s/$1.list"' % deploylist_path) + lines.append('preservedir="%s/$1.preserve"' % deploylist_path) + lines.append('if [ -f $manifest ] ; then') + # Read manifest in reverse and delete files / remove empty dirs + lines.append(' sed \'1!G;h;$!d\' $manifest | while read file') + lines.append(' do') + if dryrun: + lines.append(' if [ ! -d $file ] ; then') + lines.append(' echo $file') + lines.append(' fi') + else: + lines.append(' if [ -d $file ] ; then') + # Avoid deleting a preserved directory in case it has special perms + lines.append(' if [ ! -d $preservedir/$file ] ; then') + lines.append(' rmdir $file > /dev/null 2>&1 || true') + lines.append(' fi') + lines.append(' else') + lines.append(' rm $file') + lines.append(' fi') + lines.append(' done') + if not dryrun: + lines.append(' rm $manifest') + if not deploy and not dryrun: + # May as well remove all traces + lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true') + lines.append('fi') + + if deploy: + if not nocheckspace: + # Check for available space + # FIXME This doesn't take into account files spread across multiple + # partitions, but doing that is non-trivial + # Find the part of the destination path that exists + lines.append('checkpath="$2"') + lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]') + lines.append('do') + lines.append(' checkpath=`dirname "$checkpath"`') + lines.append('done') + lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') + # First line of the file is the total space + lines.append('total=`head -n1 $3`') + lines.append('if [ $total -gt $freespace ] ; then') + lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') + lines.append(' exit 1') + lines.append('fi') + if not nopreserve: + # Preserve any files that exist. Note that this will add to the + # preserved list with successive deployments if the list of files + # deployed changes, but because we've deleted any previously + # deployed files at this point it will never preserve anything + # that was deployed, only files that existed prior to any deploying + # (which makes the most sense) + lines.append('cat $3 | sed "1d" | while read file fsize') + lines.append('do') + lines.append(' if [ -e $file ] ; then') + lines.append(' dest="$preservedir/$file"') + lines.append(' mkdir -p `dirname $dest`') + lines.append(' mv $file $dest') + lines.append(' fi') + lines.append('done') + lines.append('rm $3') + lines.append('mkdir -p `dirname $manifest`') + lines.append('mkdir -p $2') + if verbose: + lines.append(' tar xv -C $2 -f - | tee $manifest') + else: + lines.append(' tar xv -C $2 -f - > $manifest') + lines.append('sed -i "s!^./!$2!" $manifest') + elif not dryrun: + # Put any preserved files back + lines.append('if [ -d $preservedir ] ; then') + lines.append(' cd $preservedir') + lines.append(' find . -type f -exec mv {} /{} \;') + lines.append(' cd /') + lines.append(' rm -rf $preservedir') + lines.append('fi') + + if undeployall: + if not dryrun: + lines.append('echo "NOTE: Successfully undeployed $1"') + lines.append('done') + + # Delete the script itself + lines.append('rm $0') + lines.append('') + + return '\n'.join(lines) def deploy(args, config, basepath, workspace): + """Entry point for the devtool 'deploy' subcommand""" import re - from devtool import exec_build_env_command + import math + import oe.recipeutils + + check_workspace_recipe(workspace, args.recipename, checksrc=False) - if not args.recipename in workspace: - logger.error("no recipe named %s in your workspace" % args.recipename) - return -1 try: host, destdir = args.target.split(':') except ValueError: destdir = '/' else: args.target = host + if not destdir.endswith('/'): + destdir += '/' - deploy_dir = os.path.join(basepath, 'target_deploy', args.target) - deploy_file = os.path.join(deploy_dir, args.recipename + '.list') - - stdout, stderr = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True) - recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1) - if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): - logger.error('No files to deploy - have you built the %s recipe? If so, the install step has not installed any files.' % args.recipename) - return -1 + tinfoil = setup_tinfoil(basepath=basepath) + try: + try: + rd = tinfoil.parse_recipe(args.recipename) + except Exception as e: + raise DevtoolError('Exception parsing recipe %s: %s' % + (args.recipename, e)) + recipe_outdir = rd.getVar('D') + if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): + raise DevtoolError('No files to deploy - have you built the %s ' + 'recipe? If so, the install step has not installed ' + 'any files.' % args.recipename) - if args.dry_run: - print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) - for root, dirs, files in os.walk(recipe_outdir): + filelist = [] + ftotalsize = 0 + for root, _, files in os.walk(recipe_outdir): for fn in files: - print(' %s' % os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)) - return 0 + # Get the size in kiB (since we'll be comparing it to the output of du -k) + # MUST use lstat() here not stat() or getfilesize() since we don't want to + # dereference symlinks + fsize = int(math.ceil(float(os.lstat(os.path.join(root, fn)).st_size)/1024)) + ftotalsize += fsize + # The path as it would appear on the target + fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) + filelist.append((fpath, fsize)) - if os.path.exists(deploy_file): - if undeploy(args, config, basepath, workspace): - # Error already shown - return -1 + if args.dry_run: + print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) + for item, _ in filelist: + print(' %s' % item) + return 0 - extraoptions = '' - if args.no_host_check: - extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' - if not args.show_status: - extraoptions += ' -q' - ret = subprocess.call('scp -r %s %s/* %s:%s' % (extraoptions, recipe_outdir, args.target, destdir), shell=True) - if ret != 0: - logger.error('Deploy failed - rerun with -s to get a complete error message') - return ret - logger.info('Successfully deployed %s' % recipe_outdir) + extraoptions = '' + if args.no_host_check: + extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + if not args.show_status: + extraoptions += ' -q' - if not os.path.exists(deploy_dir): - os.makedirs(deploy_dir) + scp_port = '' + ssh_port = '' + if not args.port: + raise DevtoolError("If you specify -P/--port then you must provide the port to be used to connect to the target") + else: + scp_port = "-P %s" % args.port + ssh_port = "-p %s" % args.port - files_list = [] - for root, _, files in os.walk(recipe_outdir): - for filename in files: - filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) - f |
