diff options
Diffstat (limited to 'scripts/lib/devtool/standard.py')
| -rw-r--r-- | scripts/lib/devtool/standard.py | 615 |
1 files changed, 446 insertions, 169 deletions
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index baef23e467..5ff1e230fd 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py @@ -30,7 +30,7 @@ import errno import glob import filecmp from collections import OrderedDict -from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, DevtoolError +from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, ensure_npm, DevtoolError from devtool import parse_recipe logger = logging.getLogger('devtool') @@ -47,13 +47,13 @@ def add(args, config, basepath, workspace): # These are positional arguments, but because we're nice, allow # specifying e.g. source tree without name, or fetch URI without name or # source tree (if we can detect that that is what the user meant) - if '://' in args.recipename: + if scriptutils.is_src_url(args.recipename): if not args.fetchuri: if args.fetch: raise DevtoolError('URI specified as positional argument as well as -f/--fetch') args.fetchuri = args.recipename args.recipename = '' - elif args.srctree and '://' in args.srctree: + elif scriptutils.is_src_url(args.srctree): if not args.fetchuri: if args.fetch: raise DevtoolError('URI specified as positional argument as well as -f/--fetch') @@ -64,7 +64,7 @@ def add(args, config, basepath, workspace): args.srctree = args.recipename args.recipename = None elif os.path.isdir(args.recipename): - logger.warn('Ambiguous argument %s - assuming you mean it to be the recipe name') + logger.warn('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) if args.srctree and os.path.isfile(args.srctree): args.fetchuri = 'file://' + os.path.abspath(args.srctree) @@ -74,7 +74,7 @@ def add(args, config, basepath, workspace): if args.fetchuri: raise DevtoolError('URI specified as positional argument as well as -f/--fetch') else: - # FIXME should show a warning that -f/--fetch is deprecated here + logger.warn('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') args.fetchuri = args.fetch if args.recipename: @@ -85,10 +85,6 @@ def add(args, config, basepath, workspace): if reason: raise DevtoolError(reason) - # FIXME this ought to be in validate_pn but we're using that in other contexts - if '/' in args.recipename: - raise DevtoolError('"/" is not a valid character in recipe names') - if args.srctree: srctree = os.path.abspath(args.srctree) srctreeparent = None @@ -132,6 +128,9 @@ def add(args, config, basepath, workspace): color = args.color extracmdopts = '' if args.fetchuri: + if args.fetchuri.startswith('npm://'): + ensure_npm(config, basepath, args.fixed_setup) + source = args.fetchuri if srctree: extracmdopts += ' -x %s' % srctree @@ -151,16 +150,33 @@ def add(args, config, basepath, workspace): extracmdopts += ' --src-subdir "%s"' % args.src_subdir if args.autorev: extracmdopts += ' -a' + if args.fetch_dev: + extracmdopts += ' --fetch-dev' tempdir = tempfile.mkdtemp(prefix='devtool') try: - try: - stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, tempdir, source, extracmdopts)) - except bb.process.ExecutionError as e: - if e.exitcode == 15: - raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') - else: - raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) + builtnpm = False + while True: + try: + stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) + except bb.process.ExecutionError as e: + if e.exitcode == 14: + if builtnpm: + raise DevtoolError('Re-running recipetool still failed to find npm') + # FIXME this is a horrible hack that is unfortunately + # necessary due to the fact that we can't run bitbake from + # inside recipetool since recipetool keeps tinfoil active + # with references to it throughout the code, so we have + # to exit out and come back here to do it. + ensure_npm(config, basepath, args.fixed_setup, check_exists=False) + logger.info('Re-running recipe creation process after building nodejs') + builtnpm = True + continue + elif e.exitcode == 15: + raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') + else: + raise DevtoolError('Command \'%s\' failed' % e.command) + break recipes = glob.glob(os.path.join(tempdir, '*.bb')) if recipes: @@ -214,8 +230,17 @@ def add(args, config, basepath, workspace): tinfoil = setup_tinfoil(config_only=True, basepath=basepath) try: - rd = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, None) + try: + rd = tinfoil.parse_recipe_file(recipefile, False) + except Exception as e: + logger.error(str(e)) + rd = None if not rd: + # Parsing failed. We just created this recipe and we shouldn't + # leave it in the workdir or it'll prevent bitbake from starting + movefn = '%s.parsefailed' % recipefile + logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) + shutil.move(recipefile, movefn) return 1 if args.fetchuri and not args.no_git: @@ -293,7 +318,7 @@ def _check_compatible_recipe(pn, d): raise DevtoolError("The %s recipe is a meta-recipe, and therefore is " "not supported by this tool" % pn, 4) - if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC', True): + if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): # Not an incompatibility error per se, so we don't pass the error code raise DevtoolError("externalsrc is currently enabled for the %s " "recipe. This prevents the normal do_patch task " @@ -322,10 +347,11 @@ def _git_ls_tree(repodir, treeish='HEAD', recursive=False): cmd.append('-r') out, _ = bb.process.run(cmd, cwd=repodir) ret = {} - for line in out.split('\0'): - if line: - split = line.split(None, 4) - ret[split[3]] = split[0:3] + if out: + for line in out.split('\0'): + if line: + split = line.split(None, 4) + ret[split[3]] = split[0:3] return ret def _git_exclude_path(srctree, path): @@ -367,7 +393,7 @@ def extract(args, config, basepath, workspace): return 1 srctree = os.path.abspath(args.srctree) - initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd) + initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil) logger.info('Source tree extracted to %s' % srctree) if initial_rev: @@ -391,7 +417,7 @@ def sync(args, config, basepath, workspace): return 1 srctree = os.path.abspath(args.srctree) - initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd) + initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd, tinfoil) logger.info('Source tree %s synchronized' % srctree) if initial_rev: @@ -401,70 +427,6 @@ def sync(args, config, basepath, workspace): finally: tinfoil.shutdown() -class BbTaskExecutor(object): - """Class for executing bitbake tasks for a recipe - - FIXME: This is very awkward. Unfortunately it's not currently easy to - properly execute tasks outside of bitbake itself, until then this has to - suffice if we are to handle e.g. linux-yocto's extra tasks - """ - - def __init__(self, rdata): - self.rdata = rdata - self.executed = [] - - def exec_func(self, func, report): - """Run bitbake task function""" - if not func in self.executed: - deps = self.rdata.getVarFlag(func, 'deps', False) - if deps: - for taskdepfunc in deps: - self.exec_func(taskdepfunc, True) - if report: - logger.info('Executing %s...' % func) - fn = self.rdata.getVar('FILE', True) - localdata = bb.build._task_data(fn, func, self.rdata) - try: - bb.build.exec_func(func, localdata) - except bb.build.FuncFailed as e: - raise DevtoolError(str(e)) - self.executed.append(func) - - -class PatchTaskExecutor(BbTaskExecutor): - def __init__(self, rdata): - import oe.patch - self.check_git = False - self.useroptions = [] - oe.patch.GitApplyTree.gitCommandUserOptions(self.useroptions, d=rdata) - super(PatchTaskExecutor, self).__init__(rdata) - - def exec_func(self, func, report): - from oe.patch import GitApplyTree - srcsubdir = self.rdata.getVar('S', True) - haspatches = False - if func == 'do_patch': - patchdir = os.path.join(srcsubdir, 'patches') - if os.path.exists(patchdir): - if os.listdir(patchdir): - haspatches = True - else: - os.rmdir(patchdir) - - super(PatchTaskExecutor, self).exec_func(func, report) - if self.check_git and os.path.exists(srcsubdir): - if func == 'do_patch': - if os.path.exists(patchdir): - shutil.rmtree(patchdir) - if haspatches: - stdout, _ = bb.process.run('git status --porcelain patches', cwd=srcsubdir) - if stdout: - bb.process.run('git checkout patches', cwd=srcsubdir) - - stdout, _ = bb.process.run('git status --porcelain', cwd=srcsubdir) - if stdout: - bb.process.run('git add .; git %s commit -a -m "Committing changes from %s\n\n%s"' % (' '.join(self.useroptions), func, GitApplyTree.ignore_commit_prefix + ' - from %s' % func), cwd=srcsubdir) - def _prep_extract_operation(config, basepath, recipename, tinfoil=None): """HACK: Ugly workaround for making sure that requirements are met when @@ -488,22 +450,11 @@ def _prep_extract_operation(config, basepath, recipename, tinfoil=None): return tinfoil -def _extract_source(srctree, keep_temp, devbranch, sync, d): +def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): """Extract sources of a recipe""" - import bb.event import oe.recipeutils - def eventfilter(name, handler, event, d): - """Bitbake event filter for devtool extract operation""" - if name == 'base_eventhandler': - return True - else: - return False - - if hasattr(bb.event, 'set_eventfilter'): - bb.event.set_eventfilter(eventfilter) - - pn = d.getVar('PN', True) + pn = d.getVar('PN') _check_compatible_recipe(pn, d) @@ -528,52 +479,105 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d): bb.utils.mkdirhier(srctree) os.rmdir(srctree) - # We don't want notes to be printed, they are too verbose - origlevel = bb.logger.getEffectiveLevel() - if logger.getEffectiveLevel() > logging.DEBUG: - bb.logger.setLevel(logging.WARNING) - initial_rev = None - tempdir = tempfile.mkdtemp(prefix='devtool') + # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary + # directory so that: + # (a) we pick up all files that get unpacked to the WORKDIR, and + # (b) we don't disturb the existing build + # However, with recipe-specific sysroots the sysroots for the recipe + # will be prepared under WORKDIR, and if we used the system temporary + # directory (i.e. usually /tmp) as used by mkdtemp by default, then + # our attempts to hardlink files into the recipe-specific sysroots + # will fail on systems where /tmp is a different filesystem, and it + # would have to fall back to copying the files which is a waste of + # time. Put the temp directory under the WORKDIR to prevent that from + # being a problem. + tempbasedir = d.getVar('WORKDIR') + bb.utils.mkdirhier(tempbasedir) + tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) try: + tinfoil.logger.setLevel(logging.WARNING) + crd = d.createCopy() # Make a subdir so we guard against WORKDIR==S workdir = os.path.join(tempdir, 'workdir') crd.setVar('WORKDIR', workdir) - crd.setVar('T', os.path.join(tempdir, 'temp')) - if not crd.getVar('S', True).startswith(workdir): + if not crd.getVar('S').startswith(workdir): # Usually a shared workdir recipe (kernel, gcc) # Try to set a reasonable default if bb.data.inherits_class('kernel', d): crd.setVar('S', '${WORKDIR}/source') else: - crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S', True))) + crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S'))) if bb.data.inherits_class('kernel', d): # We don't want to move the source to STAGING_KERNEL_DIR here crd.setVar('STAGING_KERNEL_DIR', '${S}') - task_executor = PatchTaskExecutor(crd) + is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) + if not is_kernel_yocto: + crd.setVar('PATCHTOOL', 'git') + crd.setVar('PATCH_COMMIT_FUNCTIONS', '1') + + # Apply our changes to the datastore to the server's datastore + for key in crd.localkeys(): + tinfoil.config_data.setVar('%s_pn-%s' % (key, pn), crd.getVar(key, False)) + + tinfoil.config_data.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps')) + tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp')) + tinfoil.config_data.setVar('BUILDCFG_FUNCS', '') + tinfoil.config_data.setVar('BUILDCFG_HEADER', '') + tinfoil.config_data.setVar('BB_HASH_IGNORE_MISMATCH', '1') + + tinfoil.set_event_mask(['bb.event.BuildStarted', + 'bb.event.BuildCompleted', + 'logging.LogRecord', + 'bb.command.CommandCompleted', + 'bb.command.CommandFailed', + 'bb.build.TaskStarted', + 'bb.build.TaskSucceeded', + 'bb.build.TaskFailed', + 'bb.build.TaskFailedSilent']) + + def runtask(target, task): + if tinfoil.build_file(target, task): + while True: + event = tinfoil.wait_event(0.25) + if event: + if isinstance(event, bb.command.CommandCompleted): + break + elif isinstance(event, bb.command.CommandFailed): + raise DevtoolError('Task do_%s failed: %s' % (task, event.error)) + elif isinstance(event, bb.build.TaskFailed): + raise DevtoolError('Task do_%s failed' % task) + elif isinstance(event, bb.build.TaskStarted): + logger.info('Executing %s...' % event._task) + elif isinstance(event, logging.LogRecord): + if event.levelno <= logging.INFO: + continue + logger.handle(event) - crd.setVar('EXTERNALSRC_forcevariable', '') + # we need virtual:native:/path/to/recipe if it's a BBCLASSEXTEND + fn = tinfoil.get_recipe_file(pn) + runtask(fn, 'unpack') - logger.info('Fetching %s...' % pn) - task_executor.exec_func('do_fetch', False) - logger.info('Unpacking...') - task_executor.exec_func('do_unpack', False) if bb.data.inherits_class('kernel-yocto', d): # Extra step for kernel to populate the source directory - logger.info('Doing kernel checkout...') - task_executor.exec_func('do_kernel_checkout', False) - srcsubdir = crd.getVar('S', True) + runtask(fn, 'kernel_checkout') - task_executor.check_git = True + srcsubdir = crd.getVar('S') # Move local source files into separate subdir recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(crd)] local_files = oe.recipeutils.get_recipe_local_files(crd) + + # Ignore local files with subdir={BP} + srcabspath = os.path.abspath(srcsubdir) local_files = [fname for fname in local_files if - os.path.exists(os.path.join(workdir, fname))] + os.path.exists(os.path.join(workdir, fname)) and + (srcabspath == workdir or not + os.path.join(workdir, fname).startswith(srcabspath + + os.sep))] if local_files: for fname in local_files: _move_file(os.path.join(workdir, fname), @@ -590,7 +594,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d): os.path.basename(fname) not in recipe_patches] # Force separate S so that patch files can be left out from srctree srcsubdir = tempfile.mkdtemp(dir=workdir) - crd.setVar('S', srcsubdir) + tinfoil.config_data.setVar('S_task-patch', srcsubdir) # Move source files to S for path in src_files: _move_file(os.path.join(workdir, path), @@ -608,15 +612,13 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d): "doesn't use any source or the correct source " "directory could not be determined" % pn) - setup_git_repo(srcsubdir, crd.getVar('PV', True), devbranch, d=d) + setup_git_repo(srcsubdir, crd.getVar('PV'), devbranch, d=d) (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) initial_rev = stdout.rstrip() - crd.setVar('PATCHTOOL', 'git') - logger.info('Patching...') - task_executor.exec_func('do_patch', False) + runtask(fn, 'patch') bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) @@ -624,8 +626,8 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d): if bb.data.inherits_class('kernel-yocto', d): # Store generate and store kernel config logger.info('Generating kernel config') - task_executor.exec_func('do_configure', False) - kconfig = os.path.join(crd.getVar('B', True), '.config') + runtask(fn, 'configure') + kconfig = os.path.join(crd.getVar('B'), '.config') tempdir_localdir = os.path.join(tempdir, 'oe-local-files') @@ -657,13 +659,34 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d): shutil.move(srcsubdir, srctree) + if os.path.abspath(d.getVar('S')) == os.path.abspath(d.getVar('WORKDIR')): + # If recipe extracts to ${WORKDIR}, symlink the files into the srctree + # (otherwise the recipe won't build as expected) + local_files_dir = os.path.join(srctree, 'oe-local-files') + addfiles = [] + for root, _, files in os.walk(local_files_dir): + relpth = os.path.relpath(root, local_files_dir) + if relpth != '.': + bb.utils.mkdirhier(os.path.join(srctree, relpth)) + for fn in files: + if fn == '.gitignore': + continue + destpth = os.path.join(srctree, relpth, fn) + if os.path.exists(destpth): + os.unlink(destpth) + os.symlink('oe-local-files/%s' % fn, destpth) + addfiles.append(os.path.join(relpth, fn)) + if addfiles: + bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) + useroptions = [] + oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d) + bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) + if kconfig: logger.info('Copying kernel config to srctree') shutil.copy2(kconfig, srctree) finally: - bb.logger.setLevel(origlevel) - if keep_temp: logger.info('Preserving temporary directory %s' % tempdir) else: @@ -733,7 +756,7 @@ def modify(args, config, basepath, workspace): if not rd: return 1 - pn = rd.getVar('PN', True) + pn = rd.getVar('PN') if pn != args.recipename: logger.info('Mapping %s to %s' % (args.recipename, pn)) if pn in workspace: @@ -754,8 +777,10 @@ def modify(args, config, basepath, workspace): if not tinfoil: # Error already shown return 1 + # We need to re-parse because tinfoil may have been re-initialised + rd = parse_recipe(config, tinfoil, args.recipename, True) - recipefile = rd.getVar('FILE', True) + recipefile = rd.getVar('FILE') appendfile = recipe_to_append(recipefile, config, args.wildcard) if os.path.exists(appendfile): raise DevtoolError("Another variant of recipe %s is already in your " @@ -768,7 +793,7 @@ def modify(args, config, basepath, workspace): initial_rev = None commits = [] if not args.no_extract: - initial_rev = _extract_source(srctree, False, args.branch, False, rd) + initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil) if not initial_rev: return 1 logger.info('Source tree extracted to %s' % srctree) @@ -792,8 +817,8 @@ def modify(args, config, basepath, workspace): initial_rev = stdout.rstrip() # Check that recipe isn't using a shared workdir - s = os.path.abspath(rd.getVar('S', True)) - workdir = os.path.abspath(rd.getVar('WORKDIR', True)) + s = os.path.abspath(rd.getVar('S')) + workdir = os.path.abspath(rd.getVar('WORKDIR')) if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: # Handle if S is set to a subdirectory of the source srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] @@ -836,6 +861,199 @@ def modify(args, config, basepath, workspace): return 0 + +def rename(args, config, basepath, workspace): + """Entry point for the devtool 'rename' subcommand""" + import bb + import oe.recipeutils + + check_workspace_recipe(workspace, args.recipename) + + if not (args.newname or args.version): + raise DevtoolError('You must specify a new name, a version with -V/--version, or both') + + recipefile = workspace[args.recipename]['recipefile'] + if not recipefile: + raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') + + if args.newname and args.newname != args.recipename: + reason = oe.recipeutils.validate_pn(args.newname) + if reason: + raise DevtoolError(reason) + newname = args.newname + else: + newname = args.recipename + + append = workspace[args.recipename]['bbappend'] + appendfn = os.path.splitext(os.path.basename(append))[0] + splitfn = appendfn.split('_') + if len(splitfn) > 1: + origfnver = appendfn.split('_')[1] + else: + origfnver = '' + + recipefilemd5 = None + tinfoil = setup_tinfoil(basepath=basepath, tracking=True) + try: + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + + bp = rd.getVar('BP') + bpn = rd.getVar('BPN') + if newname != args.recipename: + localdata = rd.createCopy() + localdata.setVar('PN', newname) + newbpn = localdata.getVar('BPN') + else: + newbpn = bpn + s = rd.getVar('S', False) + src_uri = rd.getVar('SRC_URI', False) + pv = rd.getVar('PV') + + # Correct variable values that refer to the upstream source - these + # values must stay the same, so if the name/version are changing then + # we need to fix them up + new_s = s + new_src_uri = src_uri + if newbpn != bpn: + # ${PN} here is technically almost always incorrect, but people do use it + new_s = new_s.replace('${BPN}', bpn) + new_s = new_s.replace('${PN}', bpn) + new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) + new_src_uri = new_src_uri.replace('${BPN}', bpn) + new_src_uri = new_src_uri.replace('${PN}', bpn) + new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) + if args.version and origfnver == pv: + new_s = new_s.replace('${PV}', pv) + new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) + new_src_uri = new_src_uri.replace('${PV}', pv) + new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) + patchfields = {} + if new_s != s: + patchfields['S'] = new_s + if new_src_uri != src_uri: + patchfields['SRC_URI'] = new_src_uri + if patchfields: + recipefilemd5 = bb.utils.md5_file(recipefile) + oe.recipeutils.patch_recipe(rd, recipefile, patchfields) + newrecipefilemd5 = bb.utils.md5_file(recipefile) + finally: + tinfoil.shutdown() + + if args.version: + newver = args.version + else: + newver = origfnver + + if newver: + newappend = '%s_%s.bbappend' % (newname, newver) + newfile = '%s_%s.bb' % (newname, newver) + else: + newappend = '%s.bbappend' % newname + newfile = '%s.bb' % newname + + oldrecipedir = os.path.dirname(recipefile) + newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) + if oldrecipedir != newrecipedir: + bb.utils.mkdirhier(newrecipedir) + + newappend = os.path.join(os.path.dirname(append), newappend) + newfile = os.path.join(newrecipedir, newfile) + + # Rename bbappend + logger.info('Renaming %s to %s' % (append, newappend)) + os.rename(append, newappend) + # Rename recipe file + logger.info('Renaming %s to %s' % (recipefile, newfile)) + os.rename(recipefile, newfile) + + # Rename source tree if it's the default path + appendmd5 = None + if not args.no_srctree: + srctree = workspace[args.recipename]['srctree'] + if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): + newsrctree = os.path.join(config.workspace_path, 'sources', newname) + logger.info('Renaming %s to %s' % (srctree, newsrctree)) + shutil.move(srctree, newsrctree) + # Correct any references (basically EXTERNALSRC*) in the .bbappend + appendmd5 = bb.utils.md5_file(newappend) + appendlines = [] + with open(newappend, 'r') as f: + for line in f: + appendlines.append(line) + with open(newappend, 'w') as f: + for line in appendlines: + if srctree in line: + line = line.replace(srctree, newsrctree) + f.write(line) + newappendmd5 = bb.utils.md5_file(newappend) + + bpndir = None + newbpndir = None + if newbpn != bpn: + bpndir = os.path.join(oldrecipedir, bpn) + if os.path.exists(bpndir): + newbpndir = os.path.join(newrecipedir, newbpn) + logger.info('Renaming %s to %s' % (bpndir, newbpndir)) + shutil.move(bpndir, newbpndir) + + bpdir = None + newbpdir = None + if newver != origfnver or newbpn != bpn: + bpdir = os.path.join(oldrecipedir, bp) + if os.path.exists(bpdir): + newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) + logger.info('Renaming %s to %s' % (bpdir, newbpdir)) + shutil.move(bpdir, newbpdir) + + if oldrecipedir != newrecipedir: + # Move any stray files and delete the old recipe directory + for entry in os.listdir(oldrecipedir): + oldpath = os.path.join(oldrecipedir, entry) + newpath = os.path.join(newrecipedir, entry) + logger.info('Renaming %s to %s' % (oldpath, newpath)) + shutil.move(oldpath, newpath) + os.rmdir(oldrecipedir) + + # Now take care of entries in .devtool_md5 + md5entries = [] + with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: + for line in f: + md5entries.append(line) + + if bpndir and newbpndir: + relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' + else: + relbpndir = None + if bpdir and newbpdir: + relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' + else: + relbpdir = None + + with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: + for entry in md5entries: + splitentry = entry.rstrip().split('|') + if len(splitentry) > 2: + if splitentry[0] == args.recipename: + splitentry[0] = newname + if splitentry[1] == os.path.relpath(append, config.workspace_path): + splitentry[1] = os.path.relpath(newappend, config.workspace_path) + if appendmd5 and splitentry[2] == appendmd5: + splitentry[2] = newappendmd5 + elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): + splitentry[1] = os.path.relpath(newfile, config.workspace_path) + if recipefilemd5 and splitentry[2] == recipefilemd5: + splitentry[2] = newrecipefilemd5 + elif relbpndir and splitentry[1].startswith(relbpndir): + splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) + elif relbpdir and splitentry[1].startswith(relbpdir): + splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) + entry = '|'.join(splitentry) + '\n' + f.write(entry) + return 0 + + def _get_patchset_revs(srctree, recipe_path, initial_rev=None): """Get initial and update rev of a recipe. These are the start point of the whole patchset and start point for the patches to be re-generated/updated. @@ -894,6 +1112,15 @@ def _remove_file_entries(srcuri, filelist): break return entries, remaining +def _replace_srcuri_entry(srcuri, filename, newentry): + """Replace entry corresponding to specified file with a new entry""" + basename = os.path.basename(filename) + for i in range(len(srcuri)): + if os.path.basename(srcuri[i].split(';')[0]) == basename: + srcuri.pop(i) + srcuri.insert(i, newentry) + break + def _remove_source_files(append, files, destpath): """Unlink existing patch files""" for path in files: @@ -918,7 +1145,7 @@ def _remove_source_files(append, files, destpath): raise -def _export_patches(srctree, rd, start_rev, destdir): +def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): """Export patches from srctree to given location. Returns three-tuple of dicts: 1. updated - patches that already exist in SRCURI @@ -947,18 +1174,44 @@ def _export_patches(srctree, rd, start_rev, destdir): # revision This does assume that people are using unique shortlog # values, but they ought to be anyway... new_basename = seqpatch_re.match(new_patch).group(2) - found = False + match_name = None for old_patch in existing_patches: old_basename = seqpatch_re.match(old_patch).group(2) - if new_basename == old_basename: - updated[new_patch] = existing_patches.pop(old_patch) - found = True - # Rename patch files - if new_patch != old_patch: - os.rename(os.path.join(destdir, new_patch), - os.path.join(destdir, old_patch)) + old_basename_splitext = os.path.splitext(old_basename) + if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: + old_patch_noext = os.path.splitext(old_patch)[0] + match_name = old_patch_noext break - if not found: + elif new_basename == old_basename: + match_name = old_patch + break + if match_name: + # Rename patch files + if new_patch != match_name: + os.rename(os.path.join(destdir, new_patch), + os.path.join(destdir, match_name)) + # Need to pop it off the list now before checking changed_revs + oldpath = existing_patches.pop(old_patch) + if changed_revs is not None: + # Avoid updating patches that have not actually changed + with open(os.path.join(destdir, match_name), 'r') as f: + firstlineitems = f.readline().split() + # Looking for "From <hash>" line + if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: + if not firstlineitems[1] in changed_revs: + continue + # Recompress if necessary + if oldpath.endswith(('.gz', '.Z')): + bb.process.run(['gzip', match_name], cwd=destdir) + if oldpath.endswith('.gz'): + match_name += '.gz' + else: + match_name += '.Z' + elif oldpath.endswith('.bz2'): + bb.process.run(['bzip2', match_name], cwd=destdir) + match_name += '.bz2' + updated[match_name] = oldpath + else: added[new_patch] = None return (updated, added, existing_patches) @@ -976,7 +1229,7 @@ def _create_kconfig_diff(srctree, rd, outfile): stdout, stderr = pipe.communicate() if pipe.returncode == 1: logger.info("Updating config fragment %s" % outfile) - with open(outfile, 'w') as fobj: + with open(outfile, 'wb') as fobj: fobj.write(stdout) elif pipe.returncode == 0: logger.info("Would remove config fragment %s" % outfile) @@ -1057,8 +1310,8 @@ def _export_local_files(srctree, rd, destdir): elif fname != '.gitignore': added[fname] = None - workdir = rd.getVar('WORKDIR', True) - s = rd.getVar('S', True) + workdir = rd.getVar('WORKDIR') + s = rd.getVar('S') if not s.endswith(os.sep): s += os.sep @@ -1080,14 +1333,14 @@ def _export_local_files(srctree, rd, destdir): def _determine_files_dir(rd): """Determine the appropriate files directory for a recipe""" - recipedir = rd.getVar('FILE_DIRNAME', True) - for entry in rd.getVar('FILESPATH', True).split(':'): + recipedir = rd.getVar('FILE_DIRNAME') + for entry in rd.getVar('FILESPATH').split(':'): relpth = os.path.relpath(entry, recipedir) if not os.sep in relpth: # One (or zero) levels below only, so we don't put anything in machine-specific directories if os.path.isdir(entry): return entry - return os.path.join(recipedir, rd.getVar('BPN', True)) + return os.path.join(recipedir, rd.getVar('BPN')) def _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remove): @@ -1095,7 +1348,7 @@ def _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remo import bb import oe.recipeutils - recipefile = rd.getVar('FILE', True) + recipefile = rd.getVar('FILE') logger.info('Updating SRCREV in recipe %s' % os.path.basename(recipefile)) # Get HEAD revision @@ -1177,7 +1430,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil import bb import oe.recipeutils - recipefile = rd.getVar('FILE', True) + recipefile = rd.getVar('FILE') append = workspace[recipename]['bbappend'] if not os.path.exists(append): raise DevtoolError('unable to find workspace bbappend for recipe %s' % @@ -1188,6 +1441,10 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil raise DevtoolError('Unable to find initial revision - please specify ' 'it with --initial-rev') + dl_dir = rd.getVar('DL_DIR') + if not dl_dir.endswith('/'): + dl_dir += '/' + tempdir = tempfile.mkdtemp(prefix='devtool') try: local_files_dir = tempfile.mkdtemp(dir=tempdir) @@ -1205,7 +1462,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil # Get updated patches from source tree patches_dir = tempfile.mkdtemp(dir=tempdir) upd_p, new_p, del_p = _export_patches(srctree, rd, update_rev, - patches_dir) + patches_dir, changed_revs) updatefiles = False updaterecipe = False destpath = None @@ -1232,6 +1489,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil logger.info('No patches or local source files needed updating') else: # Update existing files + files_dir = _determine_files_dir(rd) for basepath, path in upd_f.items(): logger.info('Updating file %s' % basepath) if os.path.isabs(basepath): @@ -1243,18 +1501,19 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil updatefiles = True for basepath, |