diff options
| -rw-r--r-- | meta/lib/oe/recipeutils.py | 57 | ||||
| -rw-r--r-- | meta/lib/oeqa/selftest/devtool.py | 157 | ||||
| -rw-r--r-- | scripts/lib/devtool/standard.py | 111 |
3 files changed, 322 insertions, 3 deletions
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index b8d481aeb8..0e7abf833b 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -2,7 +2,7 @@ # # Some code borrowed from the OE layer index # -# Copyright (C) 2013-2015 Intel Corporation +# Copyright (C) 2013-2016 Intel Corporation # import sys @@ -15,6 +15,7 @@ from . import utils import shutil import re import fnmatch +import glob from collections import OrderedDict, defaultdict @@ -450,6 +451,60 @@ def validate_pn(pn): return '' +def get_bbfile_path(d, destdir, extrapathhint=None): + """ + Determine the correct path for a recipe within a layer + Parameters: + d: Recipe-specific datastore + destdir: destination directory. Can be the path to the base of the layer or a + partial path somewhere within the layer. + extrapathhint: a path relative to the base of the layer to try + """ + import bb.cookerdata + + destdir = os.path.abspath(destdir) + destlayerdir = find_layerdir(destdir) + + # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf + confdata = d.createCopy() + confdata.setVar('BBFILES', '') + confdata.setVar('LAYERDIR', destlayerdir) + destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") + confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) + pn = d.getVar('PN', True) + + bbfilespecs = (confdata.getVar('BBFILES', True) or '').split() + if destdir == destlayerdir: + for bbfilespec in bbfilespecs: + if not bbfilespec.endswith('.bbappend'): + for match in glob.glob(bbfilespec): + splitext = os.path.splitext(os.path.basename(match)) + if splitext[1] == '.bb': + mpn = splitext[0].split('_')[0] + if mpn == pn: + return os.path.dirname(match) + + # Try to make up a path that matches BBFILES + # this is a little crude, but better than nothing + bpn = d.getVar('BPN', True) + recipefn = os.path.basename(d.getVar('FILE', True)) + pathoptions = [destdir] + if extrapathhint: + pathoptions.append(os.path.join(destdir, extrapathhint)) + if destdir == destlayerdir: + pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn)) + pathoptions.append(os.path.join(destdir, 'recipes', bpn)) + pathoptions.append(os.path.join(destdir, bpn)) + elif not destdir.endswith(('/' + pn, '/' + bpn)): + pathoptions.append(os.path.join(destdir, bpn)) + closepath = '' + for pathoption in pathoptions: + bbfilepath = os.path.join(pathoption, 'test.bb') + for bbfilespec in bbfilespecs: + if fnmatch.fnmatchcase(bbfilepath, bbfilespec): + return pathoption + return None + def get_bbappend_path(d, destlayerdir, wildcardver=False): """Determine how a bbappend for a recipe should be named and located within another layer""" diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py index 0b305c893e..974333f555 100644 --- a/meta/lib/oeqa/selftest/devtool.py +++ b/meta/lib/oeqa/selftest/devtool.py @@ -5,10 +5,11 @@ import re import shutil import tempfile import glob +import fnmatch import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, runqemu +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, runqemu, get_test_layer from oeqa.utils.decorators import testcase class DevtoolBase(oeSelfTest): @@ -1189,3 +1190,157 @@ class DevtoolTests(DevtoolBase): s = "Microsoft Made No Profit From Anyone's Zunes Yo" result = runCmd("devtool --quiet selftest-reverse \"%s\"" % s) self.assertEqual(result.output, s[::-1]) + + def _setup_test_devtool_finish_upgrade(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # Use a "real" recipe from meta-selftest + recipe = 'devtool-upgrade-test1' + oldversion = '1.5.3' + newversion = '1.6.0' + oldrecipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(oldrecipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check that recipe is not already under devtool control + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + # Do the upgrade + result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, newversion)) + # Check devtool status and make sure recipe is present + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Make a change to the source + result = runCmd('sed -i \'/^#include "pv.h"/a \\/* Here is a new comment *\\/\' src/pv/number.c', cwd=tempdir) + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertIn('M src/pv/number.c', result.output) + result = runCmd('git commit src/pv/number.c -m "Add a comment to the code"', cwd=tempdir) + # Check if patch is there + recipedir = os.path.dirname(oldrecipefile) + olddir = os.path.join(recipedir, recipe + '-' + oldversion) + patchfn = '0001-Add-a-note-line-to-the-quick-reference.patch' + self.assertTrue(os.path.exists(os.path.join(olddir, patchfn)), 'Original patch file does not exist') + return recipe, oldrecipefile, recipedir, olddir, newversion, patchfn + + def test_devtool_finish_upgrade_origlayer(self): + recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta-selftest/', recipedir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta-selftest' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + self.assertFalse(os.path.exists(oldrecipefile), 'Old recipe file should have been deleted but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(olddir, patchfn)), 'Old patch file should have been deleted but wasn\'t') + newrecipefile = os.path.join(recipedir, '%s_%s.bb' % (recipe, newversion)) + newdir = os.path.join(recipedir, recipe + '-' + newversion) + self.assertTrue(os.path.exists(newrecipefile), 'New recipe file should have been copied into existing layer but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, patchfn)), 'Patch file should have been copied into new directory but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch')), 'New patch file should have been created but wasn\'t') + + def test_devtool_finish_upgrade_otherlayer(self): + recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta-selftest/', recipedir) + # Try finish to a different layer - should create a bbappend + # This cleanup isn't strictly necessary but do it anyway just in case it goes wrong and writes to here + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + oe_core_dir = os.path.join(get_bb_var('COREBASE'), 'meta') + newrecipedir = os.path.join(oe_core_dir, 'recipes-test', 'devtool') + newrecipefile = os.path.join(newrecipedir, '%s_%s.bb' % (recipe, newversion)) + self.track_for_cleanup(newrecipedir) + result = runCmd('devtool finish %s oe-core' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + self.assertTrue(os.path.exists(oldrecipefile), 'Old recipe file should not have been deleted') + self.assertTrue(os.path.exists(os.path.join(olddir, patchfn)), 'Old patch file should not have been deleted') + newdir = os.path.join(newrecipedir, recipe + '-' + newversion) + self.assertTrue(os.path.exists(newrecipefile), 'New recipe file should have been copied into existing layer but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, patchfn)), 'Patch file should have been copied into new directory but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch')), 'New patch file should have been created but wasn\'t') + + def _setup_test_devtool_finish_modify(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + # Try modifying a recipe + self.track_for_cleanup(self.workspacedir) + recipe = 'mdadm' + oldrecipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(oldrecipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s %s' % (recipe, tempdir)) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Make a change to the source + result = runCmd('sed -i \'/^#include "mdadm.h"/a \\/* Here is a new comment *\\/\' maps.c', cwd=tempdir) + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertIn('M maps.c', result.output) + result = runCmd('git commit maps.c -m "Add a comment to the code"', cwd=tempdir) + for entry in os.listdir(recipedir): + filesdir = os.path.join(recipedir, entry) + if os.path.isdir(filesdir): + break + else: + self.fail('Unable to find recipe files directory for %s' % recipe) + return recipe, oldrecipefile, recipedir, filesdir + + def test_devtool_finish_modify_origlayer(self): + recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta/', recipedir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + expected_status = [(' M', '.*/%s$' % os.path.basename(oldrecipefile)), + ('??', '.*/.*-Add-a-comment-to-the-code.patch$')] + self._check_repo_status(recipedir, expected_status) + + def test_devtool_finish_modify_otherlayer(self): + recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta/', recipedir) + relpth = os.path.relpath(recipedir, os.path.join(get_bb_var('COREBASE'), 'meta')) + appenddir = os.path.join(get_test_layer(), relpth) + self.track_for_cleanup(appenddir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta-selftest' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains the following unexpected changes after finish:\n%s' % (recipe, result.output.strip())) + appendfile = os.path.join(appenddir, os.path.splitext(os.path.basename(oldrecipefile))[0] + '.bbappend') + self.assertTrue(os.path.exists(appendfile), 'bbappend %s should have been created but wasn\'t' % appendfile) + newdir = os.path.join(appenddir, recipe) + files = os.listdir(newdir) + foundpatch = None + for fn in files: + if fnmatch.fnmatch(fn, '*-Add-a-comment-to-the-code.patch'): + foundpatch = fn + if not foundpatch: + self.fail('No patch file created next to bbappend') + files.remove(foundpatch) + if files: + self.fail('Unexpected file(s) copied next to bbappend: %s' % ', '.join(files)) diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index 5a5995f664..9c09533b54 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py @@ -1,6 +1,6 @@ # Development tool - standard commands plugin # -# Copyright (C) 2014-2015 Intel Corporation +# Copyright (C) 2014-2016 Intel Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -1394,6 +1394,106 @@ def reset(args, config, basepath, workspace): return 0 +def _get_layer(layername, d): + """Determine the base layer path for the specified layer name/path""" + layerdirs = d.getVar('BBLAYERS', True).split() + layers = {os.path.basename(p): p for p in layerdirs} + # Provide some shortcuts + if layername.lower() in ['oe-core', 'openembedded-core']: + layerdir = layers.get('meta', None) + else: + layerdir = layers.get(layername, None) + if layerdir: + layerdir = os.path.abspath(layerdir) + return layerdir or layername + +def finish(args, config, basepath, workspace): + """Entry point for the devtool 'finish' subcommand""" + import bb + import oe.recipeutils + + check_workspace_recipe(workspace, args.recipename) + + tinfoil = setup_tinfoil(basepath=basepath, tracking=True) + try: + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + + destlayerdir = _get_layer(args.destination, tinfoil.config_data) + origlayerdir = oe.recipeutils.find_layerdir(rd.getVar('FILE', True)) + + if not os.path.isdir(destlayerdir): + raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) + + if os.path.abspath(destlayerdir) == config.workspace_path: + raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) + + # If it's an upgrade, grab the original path + origpath = None + origfilelist = None + append = workspace[args.recipename]['bbappend'] + with open(append, 'r') as f: + for line in f: + if line.startswith('# original_path:'): + origpath = line.split(':')[1].strip() + elif line.startswith('# original_files:'): + origfilelist = line.split(':')[1].split() + + if origlayerdir == config.workspace_path: + # Recipe file itself is in workspace, update it there first + appendlayerdir = None + origrelpath = None + if origpath: + origlayerpath = oe.recipeutils.find_layerdir(origpath) + if origlayerpath: + origrelpath = os.path.relpath(origpath, origlayerpath) + destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) + if not destpath: + raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) + elif destlayerdir == origlayerdir: + # Same layer, update the original recipe + appendlayerdir = None + destpath = None + else: + # Create/update a bbappend in the specified layer + appendlayerdir = destlayerdir + destpath = None + + # Remove any old files in the case of an upgrade + if origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == oe.recipeutils.find_layerdir(destlayerdir): + for fn in origfilelist: + fnp = os.path.join(origpath, fn) + try: + os.remove(fnp) + except FileNotFoundError: + pass + + # Actually update the recipe / bbappend + _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, initial_rev=args.initial_rev) + + if origlayerdir == config.workspace_path and destpath: + # Recipe file itself is in the workspace - need to move it and any + # associated files to the specified layer + logger.info('Moving recipe file to %s' % destpath) + recipedir = os.path.dirname(rd.getVar('FILE', True)) + for root, _, files in os.walk(recipedir): + for fn in files: + srcpath = os.path.join(root, fn) + relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) + destdir = os.path.abspath(os.path.join(destpath, relpth)) + bb.utils.mkdirhier(destdir) + shutil.move(srcpath, os.path.join(destdir, fn)) + + finally: + tinfoil.shutdown() + + # Everything else has succeeded, we can now reset + _reset([args.recipename], no_clean=False, config=config, basepath=basepath, workspace=workspace) + + return 0 + + def get_default_srctree(config, recipename=''): """Get the default srctree path""" srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) @@ -1481,3 +1581,12 @@ def register_commands(subparsers, context): parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') parser_reset.set_defaults(func=reset) + + parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', + description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified.', + group='working', order=-100) + parser_finish.add_argument('recipename', help='Recipe to finish') + parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') + parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') + parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') + parser_finish.set_defaults(func=finish) |