diff options
Diffstat (limited to 'scripts/lib/devtool')
| -rw-r--r-- | scripts/lib/devtool/__init__.py | 85 | ||||
| -rw-r--r-- | scripts/lib/devtool/build.py | 2 | ||||
| -rw-r--r-- | scripts/lib/devtool/build_image.py | 10 | ||||
| -rw-r--r-- | scripts/lib/devtool/deploy.py | 32 | ||||
| -rw-r--r-- | scripts/lib/devtool/package.py | 8 | ||||
| -rw-r--r-- | scripts/lib/devtool/runqemu.py | 13 | ||||
| -rw-r--r-- | scripts/lib/devtool/sdk.py | 14 | ||||
| -rw-r--r-- | scripts/lib/devtool/search.py | 4 | ||||
| -rw-r--r-- | scripts/lib/devtool/standard.py | 615 | ||||
| -rw-r--r-- | scripts/lib/devtool/upgrade.py | 30 | ||||
| -rw-r--r-- | scripts/lib/devtool/utilcmds.py | 18 |
11 files changed, 587 insertions, 244 deletions
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py index b432e3d44e..d646b0cf63 100644 --- a/scripts/lib/devtool/__init__.py +++ b/scripts/lib/devtool/__init__.py @@ -23,6 +23,7 @@ import sys import subprocess import logging import re +import codecs logger = logging.getLogger('devtool') @@ -67,10 +68,10 @@ def exec_watch(cmd, **options): cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options ) + reader = codecs.getreader('utf-8')(process.stdout) buf = '' while True: - out = process.stdout.read(1) - out = out.decode('utf-8') + out = reader.read(1, 1) if out: sys.stdout.write(out) sys.stdout.flush() @@ -86,13 +87,13 @@ def exec_watch(cmd, **options): def exec_fakeroot(d, cmd, **kwargs): """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions""" # Grab the command and check it actually exists - fakerootcmd = d.getVar('FAKEROOTCMD', True) + fakerootcmd = d.getVar('FAKEROOTCMD') if not os.path.exists(fakerootcmd): logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built') return 2 # Set up the appropriate environment newenv = dict(os.environ) - fakerootenv = d.getVar('FAKEROOTENV', True) + fakerootenv = d.getVar('FAKEROOTENV') for varvalue in fakerootenv.split(): if '=' in varvalue: splitval = varvalue.split('=', 1) @@ -113,40 +114,40 @@ def setup_tinfoil(config_only=False, basepath=None, tracking=False): import bb.tinfoil tinfoil = bb.tinfoil.Tinfoil(tracking=tracking) - tinfoil.prepare(config_only) - tinfoil.logger.setLevel(logger.getEffectiveLevel()) + try: + tinfoil.prepare(config_only) + tinfoil.logger.setLevel(logger.getEffectiveLevel()) + except bb.tinfoil.TinfoilUIException: + tinfoil.shutdown() + raise DevtoolError('Failed to start bitbake environment') + except: + tinfoil.shutdown() + raise finally: os.chdir(orig_cwd) return tinfoil -def get_recipe_file(cooker, pn): - """Find recipe file corresponding a package name""" - import oe.recipeutils - recipefile = oe.recipeutils.pn_to_recipe(cooker, pn) - if not recipefile: - skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn) - if skipreasons: - logger.error('\n'.join(skipreasons)) - else: - logger.error("Unable to find any recipe file matching %s" % pn) - return recipefile - def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True): - """Parse recipe of a package""" - import oe.recipeutils - recipefile = get_recipe_file(tinfoil.cooker, pn) - if not recipefile: - # Error already logged + """Parse the specified recipe""" + try: + recipefile = tinfoil.get_recipe_file(pn) + except bb.providers.NoProvider as e: + logger.error(str(e)) return None if appends: - append_files = tinfoil.cooker.collection.get_file_appends(recipefile) + append_files = tinfoil.get_file_appends(recipefile) if filter_workspace: # Filter out appends from the workspace append_files = [path for path in append_files if not path.startswith(config.workspace_path)] else: append_files = None - return oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, append_files) + try: + rd = tinfoil.parse_recipe_file(recipefile, appends, append_files) + except Exception as e: + logger.error(str(e)) + return None + return rd def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False): """ @@ -190,7 +191,7 @@ def use_external_build(same_dir, no_same_dir, d): logger.info('Using source tree as build directory since --same-dir specified') elif bb.data.inherits_class('autotools-brokensep', d): logger.info('Using source tree as build directory since recipe inherits autotools-brokensep') - elif d.getVar('B', True) == os.path.abspath(d.getVar('S', True)): + elif d.getVar('B') == os.path.abspath(d.getVar('S')): logger.info('Using source tree as build directory since that would be the default for this recipe') else: b_is_s = False @@ -259,3 +260,35 @@ def get_bbclassextend_targets(recipefile, pn): elif variant in ['native', 'cross', 'crosssdk']: targets.append('%s-%s' % (pn, variant)) return targets + +def ensure_npm(config, basepath, fixed_setup=False, check_exists=True): + """ + Ensure that npm is available and either build it or show a + reasonable error message + """ + if check_exists: + tinfoil = setup_tinfoil(config_only=False, basepath=basepath) + try: + rd = tinfoil.parse_recipe('nodejs-native') + nativepath = rd.getVar('STAGING_BINDIR_NATIVE') + finally: + tinfoil.shutdown() + npmpath = os.path.join(nativepath, 'npm') + build_npm = not os.path.exists(npmpath) + else: + build_npm = True + + if build_npm: + logger.info('Building nodejs-native') + try: + exec_build_env_command(config.init_path, basepath, + 'bitbake -q nodejs-native -c addto_recipe_sysroot', watch=True) + except bb.process.ExecutionError as e: + if "Nothing PROVIDES 'nodejs-native'" in e.stdout: + if fixed_setup: + msg = 'nodejs-native is required for npm but is not available within this SDK' + else: + msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs' + raise DevtoolError(msg) + else: + raise diff --git a/scripts/lib/devtool/build.py b/scripts/lib/devtool/build.py index 6be549dd59..252379e9b2 100644 --- a/scripts/lib/devtool/build.py +++ b/scripts/lib/devtool/build.py @@ -80,7 +80,7 @@ def register_commands(subparsers, context): """Register devtool subcommands from this plugin""" parser_build = subparsers.add_parser('build', help='Build a recipe', description='Builds the specified recipe using bitbake (up to and including %s)' % ', '.join(_get_build_tasks(context.config)), - group='working') + group='working', order=50) parser_build.add_argument('recipename', help='Recipe to build') parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism') parser_build.set_defaults(func=build) diff --git a/scripts/lib/devtool/build_image.py b/scripts/lib/devtool/build_image.py index ae75511dc7..e5810389be 100644 --- a/scripts/lib/devtool/build_image.py +++ b/scripts/lib/devtool/build_image.py @@ -34,8 +34,8 @@ def _get_packages(tinfoil, workspace, config): result = [] for recipe in workspace: data = parse_recipe(config, tinfoil, recipe, True) - if 'class-target' in data.getVar('OVERRIDES', True).split(':'): - if recipe in data.getVar('PACKAGES', True).split(): + if 'class-target' in data.getVar('OVERRIDES').split(':'): + if recipe in data.getVar('PACKAGES').split(): result.append(recipe) else: logger.warning("Skipping recipe %s as it doesn't produce a " @@ -95,7 +95,7 @@ def build_image_task(config, basepath, workspace, image, add_packages=None, task raise TargetNotImageError() # Get the actual filename used and strip the .bb and full path - target_basename = rd.getVar('FILE', True) + target_basename = rd.getVar('FILE') target_basename = os.path.splitext(os.path.basename(target_basename))[0] config.set('SDK', 'target_basename', target_basename) config.write() @@ -132,9 +132,9 @@ def build_image_task(config, basepath, workspace, image, add_packages=None, task afile.write('%s\n' % line) if task in ['populate_sdk', 'populate_sdk_ext']: - outputdir = rd.getVar('SDK_DEPLOY', True) + outputdir = rd.getVar('SDK_DEPLOY') else: - outputdir = rd.getVar('DEPLOY_DIR_IMAGE', True) + outputdir = rd.getVar('DEPLOY_DIR_IMAGE') tmp_tinfoil = tinfoil tinfoil = None diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py index fb84f2dd08..b3730ae833 100644 --- a/scripts/lib/devtool/deploy.py +++ b/scripts/lib/devtool/deploy.py @@ -85,7 +85,7 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals lines.append('do') lines.append(' checkpath=`dirname "$checkpath"`') lines.append('done') - lines.append('freespace=`df -P $checkpath | sed "1d" | awk \'{ print $4 }\'`') + lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') # First line of the file is the total space lines.append('total=`head -n1 $3`') lines.append('if [ $total -gt $freespace ] ; then') @@ -156,11 +156,11 @@ def deploy(args, config, basepath, workspace): tinfoil = setup_tinfoil(basepath=basepath) try: try: - rd = oe.recipeutils.parse_recipe_simple(tinfoil.cooker, args.recipename, tinfoil.config_data) + rd = tinfoil.parse_recipe(args.recipename) except Exception as e: raise DevtoolError('Exception parsing recipe %s: %s' % (args.recipename, e)) - recipe_outdir = rd.getVar('D', True) + recipe_outdir = rd.getVar('D') if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): raise DevtoolError('No files to deploy - have you built the %s ' 'recipe? If so, the install step has not installed ' @@ -192,6 +192,14 @@ def deploy(args, config, basepath, workspace): if not args.show_status: extraoptions += ' -q' + scp_port = '' + ssh_port = '' + if not args.port: + raise DevtoolError("If you specify -P/--port then you must provide the port to be used to connect to the target") + else: + scp_port = "-P %s" % args.port + ssh_port = "-p %s" % args.port + # In order to delete previously deployed files and have the manifest file on # the target, we write out a shell script and then copy it to the target # so we can then run it (piping tar output to it). @@ -213,7 +221,7 @@ def deploy(args, config, basepath, workspace): for fpath, fsize in filelist: f.write('%s %d\n' % (fpath, fsize)) # Copy them to the target - ret = subprocess.call("scp %s %s/* %s:%s" % (extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + ret = subprocess.call("scp %s %s %s/* %s:%s" % (scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) if ret != 0: raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 'get a complete error message' % args.target) @@ -221,7 +229,7 @@ def deploy(args, config, basepath, workspace): shutil.rmtree(tmpdir) # Now run the script - ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s \'sh %s %s %s %s\'' % (extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) + ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s %s \'sh %s %s %s %s\'' % (ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) if ret != 0: raise DevtoolError('Deploy failed - rerun with -s to get a complete ' 'error message') @@ -251,6 +259,14 @@ def undeploy(args, config, basepath, workspace): if not args.show_status: extraoptions += ' -q' + scp_port = '' + ssh_port = '' + if not args.port: + raise DevtoolError("If you specify -P/--port then you must provide the port to be used to connect to the target") + else: + scp_port = "-P %s" % args.port + ssh_port = "-p %s" % args.port + args.target = args.target.split(':')[0] tmpdir = tempfile.mkdtemp(prefix='devtool') @@ -261,7 +277,7 @@ def undeploy(args, config, basepath, workspace): with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: f.write(shellscript) # Copy it to the target - ret = subprocess.call("scp %s %s/* %s:%s" % (extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + ret = subprocess.call("scp %s %s %s/* %s:%s" % (scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) if ret != 0: raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 'get a complete error message' % args.target) @@ -269,7 +285,7 @@ def undeploy(args, config, basepath, workspace): shutil.rmtree(tmpdir) # Now run the script - ret = subprocess.call('ssh %s %s \'sh %s %s\'' % (extraoptions, args.target, tmpscript, args.recipename), shell=True) + ret = subprocess.call('ssh %s %s %s \'sh %s %s\'' % (ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True) if ret != 0: raise DevtoolError('Undeploy failed - rerun with -s to get a complete ' 'error message') @@ -292,6 +308,7 @@ def register_commands(subparsers, context): parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true') parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true') + parser_deploy.add_argument('-P', '--port', default='22', help='Port to use for connection to the target') parser_deploy.set_defaults(func=deploy) parser_undeploy = subparsers.add_parser('undeploy-target', @@ -304,4 +321,5 @@ def register_commands(subparsers, context): parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true') + parser_undeploy.add_argument('-P', '--port', default='22', help='Port to use for connection to the target') parser_undeploy.set_defaults(func=undeploy) diff --git a/scripts/lib/devtool/package.py b/scripts/lib/devtool/package.py index afb5809a36..af9e8f15f5 100644 --- a/scripts/lib/devtool/package.py +++ b/scripts/lib/devtool/package.py @@ -28,15 +28,13 @@ def package(args, config, basepath, workspace): """Entry point for the devtool 'package' subcommand""" check_workspace_recipe(workspace, args.recipename) - tinfoil = setup_tinfoil(basepath=basepath) + tinfoil = setup_tinfoil(basepath=basepath, config_only=True) try: - tinfoil.prepare(config_only=True) - image_pkgtype = config.get('Package', 'image_pkgtype', '') if not image_pkgtype: - image_pkgtype = tinfoil.config_data.getVar('IMAGE_PKGTYPE', True) + image_pkgtype = tinfoil.config_data.getVar('IMAGE_PKGTYPE') - deploy_dir_pkg = tinfoil.config_data.getVar('DEPLOY_DIR_%s' % image_pkgtype.upper(), True) + deploy_dir_pkg = tinfoil.config_data.getVar('DEPLOY_DIR_%s' % image_pkgtype.upper()) finally: tinfoil.shutdown() diff --git a/scripts/lib/devtool/runqemu.py b/scripts/lib/devtool/runqemu.py index 303abcae4f..e26cf28c2f 100644 --- a/scripts/lib/devtool/runqemu.py +++ b/scripts/lib/devtool/runqemu.py @@ -31,8 +31,10 @@ def runqemu(args, config, basepath, workspace): tinfoil = setup_tinfoil(config_only=True, basepath=basepath) try: - machine = tinfoil.config_data.getVar('MACHINE', True) - bindir_native = tinfoil.config_data.getVar('STAGING_BINDIR_NATIVE', True) + machine = tinfoil.config_data.getVar('MACHINE') + bindir_native = os.path.join(tinfoil.config_data.getVar('STAGING_DIR'), + tinfoil.config_data.getVar('BUILD_ARCH'), + tinfoil.config_data.getVar('bindir_native').lstrip(os.path.sep)) finally: tinfoil.shutdown() @@ -48,7 +50,12 @@ def runqemu(args, config, basepath, workspace): raise DevtoolError('Unable to determine image name to run, please specify one') try: - exec_build_env_command(config.init_path, basepath, 'runqemu %s %s %s' % (machine, imagename, " ".join(args.args)), watch=True) + # FIXME runqemu assumes that if OECORE_NATIVE_SYSROOT is set then it shouldn't + # run bitbake to find out the values of various environment variables, which + # isn't the case for the extensible SDK. Work around it for now. + newenv = dict(os.environ) + newenv.pop('OECORE_NATIVE_SYSROOT', '') + exec_build_env_command(config.init_path, basepath, 'runqemu %s %s %s' % (machine, imagename, " ".join(args.args)), watch=True, env=newenv) except bb.process.ExecutionError as e: # We've already seen the output since watch=True, so just ensure we return something to the user return e.exitcode diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py index 922277b79f..e8bf0ad98c 100644 --- a/scripts/lib/devtool/sdk.py +++ b/scripts/lib/devtool/sdk.py @@ -132,9 +132,9 @@ def sdk_update(args, config, basepath, workspace): # Grab variable values tinfoil = setup_tinfoil(config_only=True, basepath=basepath) try: - stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR', True) - sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS', True) - site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION', True) + stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR') + sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS') + site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION') finally: tinfoil.shutdown() @@ -273,7 +273,7 @@ def sdk_install(args, config, basepath, workspace): rd = parse_recipe(config, tinfoil, recipe, True) if not rd: return 1 - stampprefixes[recipe] = '%s.%s' % (rd.getVar('STAMP', True), tasks[0]) + stampprefixes[recipe] = '%s.%s' % (rd.getVar('STAMP'), tasks[0]) if checkstamp(recipe): logger.info('%s is already installed' % recipe) else: @@ -306,6 +306,12 @@ def sdk_install(args, config, basepath, workspace): if failed: return 2 + try: + exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots', watch=True) + except bb.process.ExecutionError as e: + raise DevtoolError('Failed to bitbake build-sysroots:\n%s' % (str(e))) + + def register_commands(subparsers, context): """Register devtool subcommands from the sdk plugin""" if context.fixed_setup: diff --git a/scripts/lib/devtool/search.py b/scripts/lib/devtool/search.py index b44bed7f6f..054985b85d 100644 --- a/scripts/lib/devtool/search.py +++ b/scripts/lib/devtool/search.py @@ -31,7 +31,7 @@ def search(args, config, basepath, workspace): tinfoil = setup_tinfoil(config_only=False, basepath=basepath) try: - pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') defsummary = tinfoil.config_data.getVar('SUMMARY', False) or '' keyword_rc = re.compile(args.keyword) @@ -70,7 +70,7 @@ def search(args, config, basepath, workspace): if match: rd = parse_recipe(config, tinfoil, fn, True) - summary = rd.getVar('SUMMARY', True) + summary = rd.getVar('SUMMARY') if summary == rd.expand(defsummary): summary = '' print("%s %s" % (fn.ljust(20), summary)) diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index baef23e467..5ff1e230fd 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py @@ -30,7 +30,7 @@ import errno import glob import filecmp from collections import OrderedDict -from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, DevtoolError +from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, ensure_npm, DevtoolError from devtool import parse_recipe logger = logging.getLogger('devtool') @@ -47,13 +47,13 @@ def add(args, config, basepath, workspace): # These are positional arguments, but because we're nice, allow # specifying e.g. source tree without name, or fetch URI without name or # source tree (if we can detect that that is what the user meant) - if '://' in args.recipename: + if scriptutils.is_src_url(args.recipename): if not args.fetchuri: if args.fetch: raise DevtoolError('URI specified as positional argument as well as -f/--fetch') args.fetchuri = args.recipename args.recipename = '' - elif args.srctree and '://' in args.srctree: + elif scriptutils.is_src_url(args.srctree): if not args.fetchuri: if args.fetch: raise DevtoolError('URI specified as positional argument as well as -f/--fetch') @@ -64,7 +64,7 @@ def add(args, config, basepath, workspace): args.srctree = args.recipename args.recipename = None elif os.path.isdir(args.recipename): - logger.warn('Ambiguous argument %s - assuming you mean it to be the recipe name') + logger.warn('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) if args.srctree and os.path.isfile(args.srctree): args.fetchuri = 'file://' + os.path.abspath(args.srctree) @@ -74,7 +74,7 @@ def add(args, config, basepath, workspace): if args.fetchuri: raise DevtoolError('URI specified as positional argument as well as -f/--fetch') else: - # FIXME should show a warning that -f/--fetch is deprecated here + logger.warn('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') args.fetchuri = args.fetch if args.recipename: @@ -85,10 +85,6 @@ def add(args, config, basepath, workspace): if reason: raise DevtoolError(reason) - # FIXME this ought to be in validate_pn but we're using that in other contexts - if '/' in args.recipename: - raise DevtoolError('"/" is not a valid character in recipe names') - if args.srctree: srctree = os.path.abspath(args.srctree) srctreeparent = None @@ -132,6 +128,9 @@ def add(args, config, basepath, workspace): color = args.color extracmdopts = '' if args.fetchuri: + if args.fetchuri.startswith('npm://'): + ensure_npm(config, basepath, args.fixed_setup) + source = args.fetchuri if srctree: extracmdopts += ' -x %s' % srctree @@ -151,16 +150,33 @@ def add(args, config, basepath, workspace): extracmdopts += ' --src-subdir "%s"' % args.src_subdir if args.autorev: extracmdopts += ' -a' + if args.fetch_dev: + extracmdopts += ' --fetch-dev' tempdir = tempfile.mkdtemp(prefix='devtool') try: - try: - stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, tempdir, source, extracmdopts)) - except bb.process.ExecutionError as e: - if e.exitcode == 15: - raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') - else: - raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) + builtnpm = False + while True: + try: + stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) + except bb.process.ExecutionError as e: + if e.exitcode == 14: + if builtnpm: + raise DevtoolError('Re-running recipetool still failed to find npm') + # FIXME this is a horrible hack that is unfortunately + # necessary due to the fact that we can't run bitbake from + # inside recipetool since recipetool keeps tinfoil active + # with references to it throughout the code, so we have + # to exit out and come back here to do it. + ensure_npm(config, basepath, args.fixed_setup, check_exists=False) + logger.info('Re-running recipe creation process after building nodejs') + builtnpm = True + continue + elif e.exitcode == 15: + raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') + else: + raise DevtoolError('Command \'%s\' failed' % e.command) + break recipes = glob.glob(os.path.join(tempdir, '*.bb')) if recipes: @@ -214,8 +230,17 @@ def add(args, config, basepath, workspace): tinfoil = setup_tinfoil(config_only=True, basepath=basepath) try: - rd = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, None) + try: + rd = tinfoil.parse_recipe_file(recipefile, False) + except Exception as e: + logger.error(str(e)) + rd = None if not rd: + # Parsing failed. We just created this recipe and we shouldn't + # leave it in the workdir or it'll prevent bitbake from starting + movefn = '%s.parsefailed' % recipefile + logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) + shutil.move(recipefile, movefn) return 1 if args.fetchuri and not args.no_git: @@ -293,7 +318,7 @@ def _check_compatible_recipe(pn, d): raise DevtoolError("The %s recipe is a meta-recipe, and therefore is " "not supported by this tool" % pn, 4) - if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC', True): + if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): # Not an incompatibility error per se, so we don't pass the error code raise DevtoolError("externalsrc is currently enabled for the %s " "recipe. This prevents the normal do_patch task " @@ -322,10 +347,11 @@ def _git_ls_tree(repodir, treeish='HEAD', recursive=False): cmd.append('-r') out, _ = bb.process.run(cmd, cwd=repodir) ret = {} - for line in out.split('\0'): - if line: - split = line.split(None, 4) - ret[split[3]] = split[0:3] + if out: + for line in out.split('\0'): + if line: + split = line.split(None, 4) + ret[split[3]] = split[0:3] return ret def _git_exclude_path(srctree, path): @@ -367,7 +393,7 @@ def extract(args, config, basepath, workspace): return 1 srctree = os.path.abspath(args.srctree) - initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd) + initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil) logger.info('Source tree extracted to %s' % srctree) if initial_rev: @@ -391,7 +417,7 @@ def sync(args, config, basepath, workspace): return 1 srctree = os.path.abspath(args.srctree) - initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd) + initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd, tinfoil) logger.info('Source tree %s synchronized' % srctree) if initial_rev: @@ -401,70 +427,6 @@ def sync(args, config, basepath, workspace): finally: tinfoil.shutdown() -class BbTaskExecutor(object): - """Class for executing bitbake tasks for a recipe - - FIXME: This is very awkward. Unfortunately it's not currently easy to - properly execute tasks outside of bitbake itself, until then this has to - suffice if we are to handle e.g. linux-yocto's extra tasks - """ - - def __init__(self, rdata): - self.rdata = rdata - self.executed = [] - - def exec_func(self, func, report): - """Run bitbake task function""" - if not func in self.executed: - deps = self.rdata.getVarFlag(func, 'deps', False) - if deps: - for taskdepfunc in deps: - self.exec_func(taskdepfunc, True) - if report: - logger.info('Executing %s...' % func) - fn = self.rdata.getVar('FILE', True) - localdata = bb.build._task_data(fn, func, self.rdata) - try: - bb.build.exec_func(func, localdata) - except bb.build.FuncFailed as e: - raise DevtoolError(str(e)) - self.executed.append(func) - - -class PatchTaskExecutor(BbTaskExecutor): - def __init__(self, rdata): - import oe.patch - self.check_git = False - self.useroptions = [] - oe.patch.GitApplyTree.gitCommandUserOptions(self.useroptions, d=rdata) - super(PatchTaskExecutor, self).__init__(rdata) - - def exec_func(self, func, report): - from oe.patch import GitApplyTree - srcsubdir = self.rdata.getVar('S', True) - haspatches = False - if func == 'do_patch': - patchdir = os.path.join(srcsubdir, 'patches') - if os.path.exists(patchdir): - if os.listdir(patchdir): - haspatches = True - else: - os.rmdir(patchdir) - - super(PatchTaskExecutor, self).exec_func(func, report) - if self.check_git and os.path.exists(srcsubdir): - if func == 'do_patch': - if os.path.exists(patchdir): - shutil.rmtree(patchdir) - if haspatches: - stdout, _ = bb.process.run('git status --porcelain patches', cwd=srcsubdir) - if stdout: - bb.process.run('git checkout patches', cwd=srcsubdir) - - stdout, _ = bb.process.run('git status --porcelain', cwd=srcsubdir) - if stdout: - bb.process.run('git add .; git %s commit -a -m "Committing changes from %s\n\n%s"' % (' '.join(self.useroptions), func, GitApplyTree.ignore_commit_prefix + ' - from %s' % func), cwd=srcsubdir) - def _prep_extract_operation(config, basepath, recipename, tinfoil=None): """HACK: Ugly workaround for making sure that requirements are met when @@ -488,22 +450,11 @@ def _prep_extract_operation(config, basepath, recipename, tinfoil=None): return tinfoil -def _extract_source(srctree, keep_temp, devbranch, sync, d): +def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): """Extract sources of a recipe""" - import bb.event import oe.recipeutils - def eventfilter(name, handler, event, d): - """Bitbake event filter for devtool extract operation""" - if name == 'base_eventhandler': - return True - else: - return False - - if hasattr(bb.event, 'set_eventfilter'): - bb.event.set_eventfilter(eventfilter) - - pn = d.getVar('PN', True) + pn = d.getVar('PN') _check_compatible_recipe(pn, d) @@ -528,52 +479,105 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d): bb.utils.mkdirhier(srctree) os.rmdir(srctree) - # We don't want notes to be printed, they are too verbose - origlevel = bb.logger.getEffectiveLevel() - if logger.getEffectiveLevel() > logging.DEBUG: - bb.logger.setLevel(logging.WARNING) - initial_rev = None - tempdir = tempfile.mkdtemp(prefix='devtool') + # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary + # directory so that: + # (a) we pick up all files that get unpacked to the WORKDIR, and + # (b) we don't disturb the existing build + # However, with recipe-specific sysroots the sysroots for the recipe + # will be prepared under WORKDIR, and if we used the system temporary + # directory (i.e. usually /tmp) as used by mkdtemp by default, then + # our attempts to hardlink files into the recipe-specific sysroots + # will fail on systems where /tmp is a different filesystem, and it + # would have to fall back to copying the files which is a waste of + # time. Put the temp directory under the WORKDIR to prevent that from + # being a problem. + tempbasedir = d.getVar('WORKDIR') + bb.utils.mkdirhier(tempbasedir) + tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) try: + tinfoil.logger.setLevel(logging.WARNING) + crd = d.createCopy() # Make a subdir so we guard against WORKDIR==S workdir = os.path.join(tempdir, 'workdir') crd.setVar('WORKDIR', workdir) - crd.setVar('T', os.path.join(tempdir, 'temp')) - if not crd.getVar('S', True).startswith(workdir): + if not crd.getVar('S').startswith(workdir): # Usually a shared workdir recipe (kernel, gcc) # Try to set a reasonable default if bb.data.inherits_class('kernel', d): crd.setVar('S', '${WORKDIR}/source') else: - crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S', True))) + crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S'))) if bb.data.inherits_class('kernel', d): # We don't want to move the source to STAGING_KERNEL_DIR here crd.setVar('STAGING_KERNEL_DIR', '${S}') - task_executor = PatchTaskExecutor(crd) + is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) + if not is_kernel_yocto: + crd.setVar('PATCHTOOL', 'git') + crd.setVar('PATCH_COMMIT_FUNCTIONS', '1') + + # Apply our changes to the datastore to the server's datastore + for key in crd.localkeys(): + tinfoil.config_data.setVar('%s_pn-%s' % (key, pn), crd.getVar(key, False)) + + tinfoil.config_data.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps')) + tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp')) + tinfoil.config_data.setVar('BUILDCFG_FUNCS', '') + tinfoil.config_data.setVar('BUILDCFG_HEADER', '') + tinfoil.config_data.setVar('BB_HASH_IGNORE_MISMATCH', '1') + + tinfoil.set_event_mask(['bb.event.BuildStarted', + 'bb.event.BuildCompleted', + 'logging.LogRecord', + 'bb.command.CommandCompleted', + 'bb.command.CommandFailed', + 'bb.build.TaskStarted', + 'bb.build.TaskSucceeded', + 'bb.build.TaskFailed', + 'bb.build.TaskFailedSilent']) + + def runtask(target, task): + if tinfoil.build_file(target, task): + while True: |