summaryrefslogtreecommitdiff
path: root/scripts/lib/devtool
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib/devtool')
-rw-r--r--scripts/lib/devtool/__init__.py236
-rw-r--r--scripts/lib/devtool/build.py86
-rw-r--r--scripts/lib/devtool/build_image.py174
-rw-r--r--scripts/lib/devtool/build_sdk.py65
-rw-r--r--scripts/lib/devtool/deploy.py323
-rw-r--r--scripts/lib/devtool/package.py60
-rw-r--r--scripts/lib/devtool/runqemu.py74
-rw-r--r--scripts/lib/devtool/sdk.py336
-rw-r--r--scripts/lib/devtool/search.py88
-rw-r--r--scripts/lib/devtool/standard.py2150
-rw-r--r--scripts/lib/devtool/upgrade.py409
-rw-r--r--scripts/lib/devtool/utilcmds.py233
12 files changed, 3667 insertions, 567 deletions
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index 4c0d5397dc..d646b0cf63 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Development tool - utility functions for plugins
#
@@ -16,16 +16,26 @@
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
+"""Devtool plugins module"""
import os
import sys
import subprocess
import logging
+import re
+import codecs
logger = logging.getLogger('devtool')
+class DevtoolError(Exception):
+ """Exception for handling devtool errors"""
+ def __init__(self, message, exitcode=1):
+ super(DevtoolError, self).__init__(message)
+ self.exitcode = exitcode
+
+
def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
+ """Run a program in bitbake build context"""
import bb
if not 'cwd' in options:
options["cwd"] = builddir
@@ -49,17 +59,19 @@ def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
return bb.process.run('%s%s' % (init_prefix, cmd), **options)
def exec_watch(cmd, **options):
+ """Run program with stdout shown on sys.stdout"""
import bb
- if isinstance(cmd, basestring) and not "shell" in options:
+ if isinstance(cmd, str) and not "shell" in options:
options["shell"] = True
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
)
+ reader = codecs.getreader('utf-8')(process.stdout)
buf = ''
while True:
- out = process.stdout.read(1)
+ out = reader.read(1, 1)
if out:
sys.stdout.write(out)
sys.stdout.flush()
@@ -72,17 +84,211 @@ def exec_watch(cmd, **options):
return buf, None
-def setup_tinfoil():
+def exec_fakeroot(d, cmd, **kwargs):
+ """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
+ # Grab the command and check it actually exists
+ fakerootcmd = d.getVar('FAKEROOTCMD')
+ if not os.path.exists(fakerootcmd):
+ logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
+ return 2
+ # Set up the appropriate environment
+ newenv = dict(os.environ)
+ fakerootenv = d.getVar('FAKEROOTENV')
+ for varvalue in fakerootenv.split():
+ if '=' in varvalue:
+ splitval = varvalue.split('=', 1)
+ newenv[splitval[0]] = splitval[1]
+ return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs)
+
+def setup_tinfoil(config_only=False, basepath=None, tracking=False):
+ """Initialize tinfoil api from bitbake"""
import scriptpath
- bitbakepath = scriptpath.add_bitbake_lib_path()
- if not bitbakepath:
- logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
- sys.exit(1)
-
- import bb.tinfoil
- import logging
- tinfoil = bb.tinfoil.Tinfoil()
- tinfoil.prepare(False)
- tinfoil.logger.setLevel(logging.WARNING)
+ orig_cwd = os.path.abspath(os.curdir)
+ try:
+ if basepath:
+ os.chdir(basepath)
+ bitbakepath = scriptpath.add_bitbake_lib_path()
+ if not bitbakepath:
+ logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
+ sys.exit(1)
+
+ import bb.tinfoil
+ tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
+ try:
+ tinfoil.prepare(config_only)
+ tinfoil.logger.setLevel(logger.getEffectiveLevel())
+ except bb.tinfoil.TinfoilUIException:
+ tinfoil.shutdown()
+ raise DevtoolError('Failed to start bitbake environment')
+ except:
+ tinfoil.shutdown()
+ raise
+ finally:
+ os.chdir(orig_cwd)
return tinfoil
+def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
+ """Parse the specified recipe"""
+ try:
+ recipefile = tinfoil.get_recipe_file(pn)
+ except bb.providers.NoProvider as e:
+ logger.error(str(e))
+ return None
+ if appends:
+ append_files = tinfoil.get_file_appends(recipefile)
+ if filter_workspace:
+ # Filter out appends from the workspace
+ append_files = [path for path in append_files if
+ not path.startswith(config.workspace_path)]
+ else:
+ append_files = None
+ try:
+ rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
+ except Exception as e:
+ logger.error(str(e))
+ return None
+ return rd
+
+def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
+ """
+ Check that a recipe is in the workspace and (optionally) that source
+ is present.
+ """
+
+ workspacepn = pn
+
+ for recipe, value in workspace.items():
+ if recipe == pn:
+ break
+ if bbclassextend:
+ recipefile = value['recipefile']
+ if recipefile:
+ targets = get_bbclassextend_targets(recipefile, recipe)
+ if pn in targets:
+ workspacepn = recipe
+ break
+ else:
+ raise DevtoolError("No recipe named '%s' in your workspace" % pn)
+
+ if checksrc:
+ srctree = workspace[workspacepn]['srctree']
+ if not os.path.exists(srctree):
+ raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn))
+ if not os.listdir(srctree):
+ raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn))
+
+ return workspacepn
+
+def use_external_build(same_dir, no_same_dir, d):
+ """
+ Determine if we should use B!=S (separate build and source directories) or not
+ """
+ b_is_s = True
+ if no_same_dir:
+ logger.info('Using separate build directory since --no-same-dir specified')
+ b_is_s = False
+ elif same_dir:
+ logger.info('Using source tree as build directory since --same-dir specified')
+ elif bb.data.inherits_class('autotools-brokensep', d):
+ logger.info('Using source tree as build directory since recipe inherits autotools-brokensep')
+ elif d.getVar('B') == os.path.abspath(d.getVar('S')):
+ logger.info('Using source tree as build directory since that would be the default for this recipe')
+ else:
+ b_is_s = False
+ return b_is_s
+
+def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
+ """
+ Set up the git repository for the source tree
+ """
+ import bb.process
+ import oe.patch
+ if not os.path.exists(os.path.join(repodir, '.git')):
+ bb.process.run('git init', cwd=repodir)
+ bb.process.run('git add .', cwd=repodir)
+ commit_cmd = ['git']
+ oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d)
+ commit_cmd += ['commit', '-q']
+ stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
+ if not stdout:
+ commit_cmd.append('--allow-empty')
+ commitmsg = "Initial empty commit with no upstream sources"
+ elif version:
+ commitmsg = "Initial commit from upstream at version %s" % version
+ else:
+ commitmsg = "Initial commit from upstream"
+ commit_cmd += ['-m', commitmsg]
+ bb.process.run(commit_cmd, cwd=repodir)
+
+ bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
+ bb.process.run('git tag -f %s' % basetag, cwd=repodir)
+
+def recipe_to_append(recipefile, config, wildcard=False):
+ """
+ Convert a recipe file to a bbappend file path within the workspace.
+ NOTE: if the bbappend already exists, you should be using
+ workspace[args.recipename]['bbappend'] instead of calling this
+ function.
+ """
+ appendname = os.path.splitext(os.path.basename(recipefile))[0]
+ if wildcard:
+ appendname = re.sub(r'_.*', '_%', appendname)
+ appendpath = os.path.join(config.workspace_path, 'appends')
+ appendfile = os.path.join(appendpath, appendname + '.bbappend')
+ return appendfile
+
+def get_bbclassextend_targets(recipefile, pn):
+ """
+ Cheap function to get BBCLASSEXTEND and then convert that to the
+ list of targets that would result.
+ """
+ import bb.utils
+
+ values = {}
+ def get_bbclassextend_varfunc(varname, origvalue, op, newlines):
+ values[varname] = origvalue
+ return origvalue, None, 0, True
+ with open(recipefile, 'r') as f:
+ bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc)
+
+ targets = []
+ bbclassextend = values.get('BBCLASSEXTEND', '').split()
+ if bbclassextend:
+ for variant in bbclassextend:
+ if variant == 'nativesdk':
+ targets.append('%s-%s' % (variant, pn))
+ elif variant in ['native', 'cross', 'crosssdk']:
+ targets.append('%s-%s' % (pn, variant))
+ return targets
+
+def ensure_npm(config, basepath, fixed_setup=False, check_exists=True):
+ """
+ Ensure that npm is available and either build it or show a
+ reasonable error message
+ """
+ if check_exists:
+ tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
+ try:
+ rd = tinfoil.parse_recipe('nodejs-native')
+ nativepath = rd.getVar('STAGING_BINDIR_NATIVE')
+ finally:
+ tinfoil.shutdown()
+ npmpath = os.path.join(nativepath, 'npm')
+ build_npm = not os.path.exists(npmpath)
+ else:
+ build_npm = True
+
+ if build_npm:
+ logger.info('Building nodejs-native')
+ try:
+ exec_build_env_command(config.init_path, basepath,
+ 'bitbake -q nodejs-native -c addto_recipe_sysroot', watch=True)
+ except bb.process.ExecutionError as e:
+ if "Nothing PROVIDES 'nodejs-native'" in e.stdout:
+ if fixed_setup:
+ msg = 'nodejs-native is required for npm but is not available within this SDK'
+ else:
+ msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs'
+ raise DevtoolError(msg)
+ else:
+ raise
diff --git a/scripts/lib/devtool/build.py b/scripts/lib/devtool/build.py
new file mode 100644
index 0000000000..252379e9b2
--- /dev/null
+++ b/scripts/lib/devtool/build.py
@@ -0,0 +1,86 @@
+# Development tool - build command plugin
+#
+# Copyright (C) 2014-2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""Devtool build plugin"""
+
+import os
+import bb
+import logging
+import argparse
+import tempfile
+from devtool import exec_build_env_command, check_workspace_recipe, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+
+def _set_file_values(fn, values):
+ remaining = list(values.keys())
+
+ def varfunc(varname, origvalue, op, newlines):
+ newvalue = values.get(varname, origvalue)
+ remaining.remove(varname)
+ return (newvalue, '=', 0, True)
+
+ with open(fn, 'r') as f:
+ (updated, newlines) = bb.utils.edit_metadata(f, values, varfunc)
+
+ for item in remaining:
+ updated = True
+ newlines.append('%s = "%s"' % (item, values[item]))
+
+ if updated:
+ with open(fn, 'w') as f:
+ f.writelines(newlines)
+ return updated
+
+def _get_build_tasks(config):
+ tasks = config.get('Build', 'build_task', 'populate_sysroot,packagedata').split(',')
+ return ['do_%s' % task.strip() for task in tasks]
+
+def build(args, config, basepath, workspace):
+ """Entry point for the devtool 'build' subcommand"""
+ workspacepn = check_workspace_recipe(workspace, args.recipename, bbclassextend=True)
+
+ build_tasks = _get_build_tasks(config)
+
+ bbappend = workspace[workspacepn]['bbappend']
+ if args.disable_parallel_make:
+ logger.info("Disabling 'make' parallelism")
+ _set_file_values(bbappend, {'PARALLEL_MAKE': ''})
+ try:
+ bbargs = []
+ for task in build_tasks:
+ if args.recipename.endswith('-native') and 'package' in task:
+ continue
+ bbargs.append('%s:%s' % (args.recipename, task))
+ exec_build_env_command(config.init_path, basepath, 'bitbake %s' % ' '.join(bbargs), watch=True)
+ except bb.process.ExecutionError as e:
+ # We've already seen the output since watch=True, so just ensure we return something to the user
+ return e.exitcode
+ finally:
+ if args.disable_parallel_make:
+ _set_file_values(bbappend, {'PARALLEL_MAKE': None})
+
+ return 0
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from this plugin"""
+ parser_build = subparsers.add_parser('build', help='Build a recipe',
+ description='Builds the specified recipe using bitbake (up to and including %s)' % ', '.join(_get_build_tasks(context.config)),
+ group='working', order=50)
+ parser_build.add_argument('recipename', help='Recipe to build')
+ parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism')
+ parser_build.set_defaults(func=build)
diff --git a/scripts/lib/devtool/build_image.py b/scripts/lib/devtool/build_image.py
new file mode 100644
index 0000000000..e5810389be
--- /dev/null
+++ b/scripts/lib/devtool/build_image.py
@@ -0,0 +1,174 @@
+# Development tool - build-image plugin
+#
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""Devtool plugin containing the build-image subcommand."""
+
+import os
+import errno
+import logging
+
+from bb.process import ExecutionError
+from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+class TargetNotImageError(Exception):
+ pass
+
+def _get_packages(tinfoil, workspace, config):
+ """Get list of packages from recipes in the workspace."""
+ result = []
+ for recipe in workspace:
+ data = parse_recipe(config, tinfoil, recipe, True)
+ if 'class-target' in data.getVar('OVERRIDES').split(':'):
+ if recipe in data.getVar('PACKAGES').split():
+ result.append(recipe)
+ else:
+ logger.warning("Skipping recipe %s as it doesn't produce a "
+ "package with the same name", recipe)
+ return result
+
+def build_image(args, config, basepath, workspace):
+ """Entry point for the devtool 'build-image' subcommand."""
+
+ image = args.imagename
+ auto_image = False
+ if not image:
+ sdk_targets = config.get('SDK', 'sdk_targets', '').split()
+ if sdk_targets:
+ image = sdk_targets[0]
+ auto_image = True
+ if not image:
+ raise DevtoolError('Unable to determine image to build, please specify one')
+
+ try:
+ if args.add_packages:
+ add_packages = args.add_packages.split(',')
+ else:
+ add_packages = None
+ result, outputdir = build_image_task(config, basepath, workspace, image, add_packages)
+ except TargetNotImageError:
+ if auto_image:
+ raise DevtoolError('Unable to determine image to build, please specify one')
+ else:
+ raise DevtoolError('Specified recipe %s is not an image recipe' % image)
+
+ if result == 0:
+ logger.info('Successfully built %s. You can find output files in %s'
+ % (image, outputdir))
+ return result
+
+def build_image_task(config, basepath, workspace, image, add_packages=None, task=None, extra_append=None):
+ # remove <image>.bbappend to make sure setup_tinfoil doesn't
+ # break because of it
+ target_basename = config.get('SDK', 'target_basename', '')
+ if target_basename:
+ appendfile = os.path.join(config.workspace_path, 'appends',
+ '%s.bbappend' % target_basename)
+ try:
+ os.unlink(appendfile)
+ except OSError as exc:
+ if exc.errno != errno.ENOENT:
+ raise
+
+ tinfoil = setup_tinfoil(basepath=basepath)
+ try:
+ rd = parse_recipe(config, tinfoil, image, True)
+ if not rd:
+ # Error already shown
+ return (1, None)
+ if not bb.data.inherits_class('image', rd):
+ raise TargetNotImageError()
+
+ # Get the actual filename used and strip the .bb and full path
+ target_basename = rd.getVar('FILE')
+ target_basename = os.path.splitext(os.path.basename(target_basename))[0]
+ config.set('SDK', 'target_basename', target_basename)
+ config.write()
+
+ appendfile = os.path.join(config.workspace_path, 'appends',
+ '%s.bbappend' % target_basename)
+
+ outputdir = None
+ try:
+ if workspace or add_packages:
+ if add_packages:
+ packages = add_packages
+ else:
+ packages = _get_packages(tinfoil, workspace, config)
+ else:
+ packages = None
+ if not task:
+ if not packages and not add_packages and workspace:
+ logger.warning('No recipes in workspace, building image %s unmodified', image)
+ elif not packages:
+ logger.warning('No packages to add, building image %s unmodified', image)
+
+ if packages or extra_append:
+ bb.utils.mkdirhier(os.path.dirname(appendfile))
+ with open(appendfile, 'w') as afile:
+ if packages:
+ # include packages from workspace recipes into the image
+ afile.write('IMAGE_INSTALL_append = " %s"\n' % ' '.join(packages))
+ if not task:
+ logger.info('Building image %s with the following '
+ 'additional packages: %s', image, ' '.join(packages))
+ if extra_append:
+ for line in extra_append:
+ afile.write('%s\n' % line)
+
+ if task in ['populate_sdk', 'populate_sdk_ext']:
+ outputdir = rd.getVar('SDK_DEPLOY')
+ else:
+ outputdir = rd.getVar('DEPLOY_DIR_IMAGE')
+
+ tmp_tinfoil = tinfoil
+ tinfoil = None
+ tmp_tinfoil.shutdown()
+
+ options = ''
+ if task:
+ options += '-c %s' % task
+
+ # run bitbake to build image (or specified task)
+ try:
+ exec_build_env_command(config.init_path, basepath,
+ 'bitbake %s %s' % (options, image), watch=True)
+ except ExecutionError as err:
+ return (err.exitcode, None)
+ finally:
+ if os.path.isfile(appendfile):
+ os.unlink(appendfile)
+ finally:
+ if tinfoil:
+ tinfoil.shutdown()
+ return (0, outputdir)
+
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from the build-image plugin"""
+ parser = subparsers.add_parser('build-image',
+ help='Build image including workspace recipe packages',
+ description='Builds an image, extending it to include '
+ 'packages from recipes in the workspace',
+ group='testbuild', order=-10)
+ parser.add_argument('imagename', help='Image recipe to build', nargs='?')
+ parser.add_argument('-p', '--add-packages', help='Instead of adding packages for the '
+ 'entire workspace, specify packages to be added to the image '
+ '(separate multiple packages by commas)',
+ metavar='PACKAGES')
+ parser.set_defaults(func=build_image)
diff --git a/scripts/lib/devtool/build_sdk.py b/scripts/lib/devtool/build_sdk.py
new file mode 100644
index 0000000000..b89d65b0cb
--- /dev/null
+++ b/scripts/lib/devtool/build_sdk.py
@@ -0,0 +1,65 @@
+# Development tool - build-sdk command plugin
+#
+# Copyright (C) 2015-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import subprocess
+import logging
+import glob
+import shutil
+import errno
+import sys
+import tempfile
+from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
+from devtool import build_image
+
+logger = logging.getLogger('devtool')
+
+
+def build_sdk(args, config, basepath, workspace):
+ """Entry point for the devtool build-sdk command"""
+
+ sdk_targets = config.get('SDK', 'sdk_targets', '').split()
+ if sdk_targets:
+ image = sdk_targets[0]
+ else:
+ raise DevtoolError('Unable to determine image to build SDK for')
+
+ extra_append = ['SDK_DERIVATIVE = "1"']
+ try:
+ result, outputdir = build_image.build_image_task(config,
+ basepath,
+ workspace,
+ image,
+ task='populate_sdk_ext',
+ extra_append=extra_append)
+ except build_image.TargetNotImageError:
+ raise DevtoolError('Unable to determine image to build SDK for')
+
+ if result == 0:
+ logger.info('Successfully built SDK. You can find output files in %s'
+ % outputdir)
+ return result
+
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands"""
+ if context.fixed_setup:
+ parser_build_sdk = subparsers.add_parser('build-sdk',
+ help='Build a derivative SDK of this one',
+ description='Builds an extensible SDK based upon this one and the items in your workspace',
+ group='advanced')
+ parser_build_sdk.set_defaults(func=build_sdk)
diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index f016b23dba..b3730ae833 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -1,6 +1,6 @@
# Development tool - deploy/undeploy command plugin
#
-# Copyright (C) 2014-2015 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -14,92 +14,244 @@
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""Devtool plugin containing the deploy subcommands"""
import os
import subprocess
import logging
-from devtool import exec_build_env_command
+import tempfile
+import shutil
+import argparse_oe
+from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
logger = logging.getLogger('devtool')
-def plugin_init(pluginlist):
- pass
+deploylist_path = '/.devtool'
+
+def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
+ """
+ Prepare a shell script for running on the target to
+ deploy/undeploy files. We have to be careful what we put in this
+ script - only commands that are likely to be available on the
+ target are suitable (the target might be constrained, e.g. using
+ busybox rather than bash with coreutils).
+ """
+ lines = []
+ lines.append('#!/bin/sh')
+ lines.append('set -e')
+ if undeployall:
+ # Yes, I know this is crude - but it does work
+ lines.append('for entry in %s/*.list; do' % deploylist_path)
+ lines.append('[ ! -f $entry ] && exit')
+ lines.append('set `basename $entry | sed "s/.list//"`')
+ if dryrun:
+ if not deploy:
+ lines.append('echo "Previously deployed files for $1:"')
+ lines.append('manifest="%s/$1.list"' % deploylist_path)
+ lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
+ lines.append('if [ -f $manifest ] ; then')
+ # Read manifest in reverse and delete files / remove empty dirs
+ lines.append(' sed \'1!G;h;$!d\' $manifest | while read file')
+ lines.append(' do')
+ if dryrun:
+ lines.append(' if [ ! -d $file ] ; then')
+ lines.append(' echo $file')
+ lines.append(' fi')
+ else:
+ lines.append(' if [ -d $file ] ; then')
+ # Avoid deleting a preserved directory in case it has special perms
+ lines.append(' if [ ! -d $preservedir/$file ] ; then')
+ lines.append(' rmdir $file > /dev/null 2>&1 || true')
+ lines.append(' fi')
+ lines.append(' else')
+ lines.append(' rm $file')
+ lines.append(' fi')
+ lines.append(' done')
+ if not dryrun:
+ lines.append(' rm $manifest')
+ if not deploy and not dryrun:
+ # May as well remove all traces
+ lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true')
+ lines.append('fi')
+
+ if deploy:
+ if not nocheckspace:
+ # Check for available space
+ # FIXME This doesn't take into account files spread across multiple
+ # partitions, but doing that is non-trivial
+ # Find the part of the destination path that exists
+ lines.append('checkpath="$2"')
+ lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
+ lines.append('do')
+ lines.append(' checkpath=`dirname "$checkpath"`')
+ lines.append('done')
+ lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")')
+ # First line of the file is the total space
+ lines.append('total=`head -n1 $3`')
+ lines.append('if [ $total -gt $freespace ] ; then')
+ lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
+ lines.append(' exit 1')
+ lines.append('fi')
+ if not nopreserve:
+ # Preserve any files that exist. Note that this will add to the
+ # preserved list with successive deployments if the list of files
+ # deployed changes, but because we've deleted any previously
+ # deployed files at this point it will never preserve anything
+ # that was deployed, only files that existed prior to any deploying
+ # (which makes the most sense)
+ lines.append('cat $3 | sed "1d" | while read file fsize')
+ lines.append('do')
+ lines.append(' if [ -e $file ] ; then')
+ lines.append(' dest="$preservedir/$file"')
+ lines.append(' mkdir -p `dirname $dest`')
+ lines.append(' mv $file $dest')
+ lines.append(' fi')
+ lines.append('done')
+ lines.append('rm $3')
+ lines.append('mkdir -p `dirname $manifest`')
+ lines.append('mkdir -p $2')
+ if verbose:
+ lines.append(' tar xv -C $2 -f - | tee $manifest')
+ else:
+ lines.append(' tar xv -C $2 -f - > $manifest')
+ lines.append('sed -i "s!^./!$2!" $manifest')
+ elif not dryrun:
+ # Put any preserved files back
+ lines.append('if [ -d $preservedir ] ; then')
+ lines.append(' cd $preservedir')
+ lines.append(' find . -type f -exec mv {} /{} \;')
+ lines.append(' cd /')
+ lines.append(' rm -rf $preservedir')
+ lines.append('fi')
+
+ if undeployall:
+ if not dryrun:
+ lines.append('echo "NOTE: Successfully undeployed $1"')
+ lines.append('done')
+
+ # Delete the script itself
+ lines.append('rm $0')
+ lines.append('')
+
+ return '\n'.join(lines)
def deploy(args, config, basepath, workspace):
+ """Entry point for the devtool 'deploy' subcommand"""
import re
- from devtool import exec_build_env_command
+ import math
+ import oe.recipeutils
+
+ check_workspace_recipe(workspace, args.recipename, checksrc=False)
- if not args.recipename in workspace:
- logger.error("no recipe named %s in your workspace" % args.recipename)
- return -1
try:
host, destdir = args.target.split(':')
except ValueError:
destdir = '/'
else:
args.target = host
+ if not destdir.endswith('/'):
+ destdir += '/'
- deploy_dir = os.path.join(basepath, 'target_deploy', args.target)
- deploy_file = os.path.join(deploy_dir, args.recipename + '.list')
-
- stdout, stderr = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
- recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
- if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
- logger.error('No files to deploy - have you built the %s recipe? If so, the install step has not installed any files.' % args.recipename)
- return -1
+ tinfoil = setup_tinfoil(basepath=basepath)
+ try:
+ try:
+ rd = tinfoil.parse_recipe(args.recipename)
+ except Exception as e:
+ raise DevtoolError('Exception parsing recipe %s: %s' %
+ (args.recipename, e))
+ recipe_outdir = rd.getVar('D')
+ if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
+ raise DevtoolError('No files to deploy - have you built the %s '
+ 'recipe? If so, the install step has not installed '
+ 'any files.' % args.recipename)
- if args.dry_run:
- print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
- for root, dirs, files in os.walk(recipe_outdir):
+ filelist = []
+ ftotalsize = 0
+ for root, _, files in os.walk(recipe_outdir):
for fn in files:
- print(' %s' % os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn))
- return 0
+ # Get the size in kiB (since we'll be comparing it to the output of du -k)
+ # MUST use lstat() here not stat() or getfilesize() since we don't want to
+ # dereference symlinks
+ fsize = int(math.ceil(float(os.lstat(os.path.join(root, fn)).st_size)/1024))
+ ftotalsize += fsize
+ # The path as it would appear on the target
+ fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
+ filelist.append((fpath, fsize))
- if os.path.exists(deploy_file):
- if undeploy(args, config, basepath, workspace):
- # Error already shown
- return -1
+ if args.dry_run:
+ print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
+ for item, _ in filelist:
+ print(' %s' % item)
+ return 0
- extraoptions = ''
- if args.no_host_check:
- extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
- if not args.show_status:
- extraoptions += ' -q'
- ret = subprocess.call('scp -r %s %s/* %s:%s' % (extraoptions, recipe_outdir, args.target, destdir), shell=True)
- if ret != 0:
- logger.error('Deploy failed - rerun with -s to get a complete error message')
- return ret
- logger.info('Successfully deployed %s' % recipe_outdir)
+ extraoptions = ''
+ if args.no_host_check:
+ extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+ if not args.show_status:
+ extraoptions += ' -q'
- if not os.path.exists(deploy_dir):
- os.makedirs(deploy_dir)
+ scp_port = ''
+ ssh_port = ''
+ if not args.port:
+ raise DevtoolError("If you specify -P/--port then you must provide the port to be used to connect to the target")
+ else:
+ scp_port = "-P %s" % args.port
+ ssh_port = "-p %s" % args.port
- files_list = []
- for root, _, files in os.walk(recipe_outdir):
- for filename in files:
- filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
- f