diff options
| author | Richard Purdie <rpurdie@linux.intel.com> | 2010-01-20 18:46:02 +0000 |
|---|---|---|
| committer | Richard Purdie <rpurdie@linux.intel.com> | 2010-01-20 18:46:02 +0000 |
| commit | 22c29d8651668195f72e2f6a8e059d625eb511c3 (patch) | |
| tree | dd1dd43f0ec47a9964c8a766eb8b3ad75cf51a64 /bitbake/lib/bb | |
| parent | 1bfd6edef9db9c9175058ae801d1b601e4f15263 (diff) | |
| download | openembedded-core-22c29d8651668195f72e2f6a8e059d625eb511c3.tar.gz openembedded-core-22c29d8651668195f72e2f6a8e059d625eb511c3.tar.bz2 openembedded-core-22c29d8651668195f72e2f6a8e059d625eb511c3.zip | |
bitbake: Switch to bitbake-dev version (bitbake master upstream)
Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
Diffstat (limited to 'bitbake/lib/bb')
37 files changed, 4731 insertions, 735 deletions
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py index b8f7c7f59e..f2f8f656d8 100644 --- a/bitbake/lib/bb/__init__.py +++ b/bitbake/lib/bb/__init__.py @@ -21,7 +21,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -__version__ = "1.8.13" +__version__ = "1.9.0" __all__ = [ @@ -54,6 +54,7 @@ __all__ = [ # modules "parse", "data", + "command", "event", "build", "fetch", diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py index 1d6742b6e6..6d80b4b549 100644 --- a/bitbake/lib/bb/build.py +++ b/bitbake/lib/bb/build.py @@ -25,8 +25,8 @@ # #Based on functions from the base bb module, Copyright 2003 Holger Schurig -from bb import data, fetch, event, mkdirhier, utils -import bb, os +from bb import data, event, mkdirhier, utils +import bb, os, sys # When we execute a python function we'd like certain things # in all namespaces, hence we add them to __builtins__ @@ -37,7 +37,11 @@ __builtins__['os'] = os # events class FuncFailed(Exception): - """Executed function failed""" + """ + Executed function failed + First parameter a message + Second paramter is a logfile (optional) + """ class EventException(Exception): """Exception which is associated with an Event.""" @@ -50,7 +54,9 @@ class TaskBase(event.Event): def __init__(self, t, d ): self._task = t - event.Event.__init__(self, d) + self._package = bb.data.getVar("PF", d, 1) + event.Event.__init__(self) + self._message = "package %s: task %s: %s" % (bb.data.getVar("PF", d, 1), t, bb.event.getName(self)[4:]) def getTask(self): return self._task @@ -68,6 +74,10 @@ class TaskSucceeded(TaskBase): class TaskFailed(TaskBase): """Task execution failed""" + def __init__(self, msg, logfile, t, d ): + self.logfile = logfile + self.msg = msg + TaskBase.__init__(self, t, d) class InvalidTask(TaskBase): """Invalid Task""" @@ -104,42 +114,116 @@ def exec_func(func, d, dirs = None): else: adir = data.getVar('B', d, 1) + # Save current directory try: prevdir = os.getcwd() except OSError: prevdir = data.getVar('TOPDIR', d, True) + + # Setup logfiles + t = data.getVar('T', d, 1) + if not t: + bb.msg.fatal(bb.msg.domain.Build, "T not set") + mkdirhier(t) + # Gross hack, FIXME + import random + logfile = "%s/log.%s.%s.%s" % (t, func, str(os.getpid()),random.random()) + runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) + + # Change to correct directory (if specified) if adir and os.access(adir, os.F_OK): os.chdir(adir) + # Handle logfiles + si = file('/dev/null', 'r') + try: + if bb.msg.debug_level['default'] > 0 or ispython: + so = os.popen("tee \"%s\"" % logfile, "w") + else: + so = file(logfile, 'w') + except OSError, e: + bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e) + pass + + se = so + + # Dup the existing fds so we dont lose them + osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] + oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] + ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] + + # Replace those fds with our own + os.dup2(si.fileno(), osi[1]) + os.dup2(so.fileno(), oso[1]) + os.dup2(se.fileno(), ose[1]) + locks = [] lockfiles = (data.expand(flags['lockfiles'], d) or "").split() for lock in lockfiles: locks.append(bb.utils.lockfile(lock)) - if flags['python']: - exec_func_python(func, d) - else: - exec_func_shell(func, d, flags) + try: + # Run the function + if ispython: + exec_func_python(func, d, runfile, logfile) + else: + exec_func_shell(func, d, runfile, logfile, flags) + + # Restore original directory + try: + os.chdir(prevdir) + except: + pass - for lock in locks: - bb.utils.unlockfile(lock) + finally: - if os.path.exists(prevdir): - os.chdir(prevdir) + # Unlock any lockfiles + for lock in locks: + bb.utils.unlockfile(lock) + + # Restore the backup fds + os.dup2(osi[0], osi[1]) + os.dup2(oso[0], oso[1]) + os.dup2(ose[0], ose[1]) + + # Close our logs + si.close() + so.close() + se.close() -def exec_func_python(func, d): + if os.path.exists(logfile) and os.path.getsize(logfile) == 0: + bb.msg.debug(2, bb.msg.domain.Build, "Zero size logfile %s, removing" % logfile) + os.remove(logfile) + + # Close the backup fds + os.close(osi[0]) + os.close(oso[0]) + os.close(ose[0]) + +def exec_func_python(func, d, runfile, logfile): """Execute a python BB 'function'""" - import re + import re, os bbfile = bb.data.getVar('FILE', d, 1) tmp = "def " + func + "():\n%s" % data.getVar(func, d) tmp += '\n' + func + '()' + + f = open(runfile, "w") + f.write(tmp) comp = utils.better_compile(tmp, func, bbfile) g = {} # globals g['d'] = d - utils.better_exec(comp, g, tmp, bbfile) + try: + utils.better_exec(comp, g, tmp, bbfile) + except: + (t,value,tb) = sys.exc_info() + + if t in [bb.parse.SkipPackage, bb.build.FuncFailed]: + raise + bb.msg.error(bb.msg.domain.Build, "Function %s failed" % func) + raise FuncFailed("function %s failed" % func, logfile) -def exec_func_shell(func, d, flags): +def exec_func_shell(func, d, runfile, logfile, flags): """Execute a shell BB 'function' Returns true if execution was successful. For this, it creates a bash shell script in the tmp dectory, writes the local @@ -149,23 +233,13 @@ def exec_func_shell(func, d, flags): of the directories you need created prior to execution. The last item in the list is where we will chdir/cd to. """ - import sys deps = flags['deps'] check = flags['check'] - interact = flags['interactive'] if check in globals(): if globals()[check](func, deps): return - global logfile - t = data.getVar('T', d, 1) - if not t: - return 0 - mkdirhier(t) - logfile = "%s/log.%s.%s" % (t, func, str(os.getpid())) - runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) - f = open(runfile, "w") f.write("#!/bin/sh -e\n") if bb.msg.debug_level['default'] > 0: f.write("set -x\n") @@ -177,91 +251,21 @@ def exec_func_shell(func, d, flags): os.chmod(runfile, 0775) if not func: bb.msg.error(bb.msg.domain.Build, "Function not specified") - raise FuncFailed() - - # open logs - si = file('/dev/null', 'r') - try: - if bb.msg.debug_level['default'] > 0: - so = os.popen("tee \"%s\"" % logfile, "w") - else: - so = file(logfile, 'w') - except OSError, e: - bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e) - pass - - se = so - - if not interact: - # dup the existing fds so we dont lose them - osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] - oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] - ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] - - # replace those fds with our own - os.dup2(si.fileno(), osi[1]) - os.dup2(so.fileno(), oso[1]) - os.dup2(se.fileno(), ose[1]) + raise FuncFailed("Function not specified for exec_func_shell") # execute function - prevdir = os.getcwd() if flags['fakeroot']: maybe_fakeroot = "PATH=\"%s\" fakeroot " % bb.data.getVar("PATH", d, 1) else: maybe_fakeroot = '' lang_environment = "LC_ALL=C " ret = os.system('%s%ssh -e %s' % (lang_environment, maybe_fakeroot, runfile)) - try: - os.chdir(prevdir) - except: - pass - - if not interact: - # restore the backups - os.dup2(osi[0], osi[1]) - os.dup2(oso[0], oso[1]) - os.dup2(ose[0], ose[1]) - # close our logs - si.close() - so.close() - se.close() - - if os.path.exists(logfile) and os.path.getsize(logfile) == 0: - bb.msg.debug(2, bb.msg.domain.Build, "Zero size logfile %s, removing" % logfile) - os.remove(logfile) - - # close the backup fds - os.close(osi[0]) - os.close(oso[0]) - os.close(ose[0]) - - if ret==0: - if bb.msg.debug_level['default'] > 0: - os.remove(runfile) -# os.remove(logfile) + if ret == 0: return - else: - bb.msg.error(bb.msg.domain.Build, "function %s failed" % func) - if data.getVar("BBINCLUDELOGS", d): - bb.msg.error(bb.msg.domain.Build, "log data follows (%s)" % logfile) - number_of_lines = data.getVar("BBINCLUDELOGS_LINES", d) - if number_of_lines: - os.system('tail -n%s %s' % (number_of_lines, logfile)) - elif os.path.exists(logfile): - f = open(logfile, "r") - while True: - l = f.readline() - if l == '': - break - l = l.rstrip() - print '| %s' % l - f.close() - else: - bb.msg.error(bb.msg.domain.Build, "There was no logfile output") - else: - bb.msg.error(bb.msg.domain.Build, "see log in %s" % logfile) - raise FuncFailed( logfile ) + + bb.msg.error(bb.msg.domain.Build, "Function %s failed" % func) + raise FuncFailed("function %s failed" % func, logfile) def exec_task(task, d): @@ -282,14 +286,20 @@ def exec_task(task, d): data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata) data.update_data(localdata) data.expandKeys(localdata) - event.fire(TaskStarted(task, localdata)) + event.fire(TaskStarted(task, localdata), localdata) exec_func(task, localdata) - event.fire(TaskSucceeded(task, localdata)) - except FuncFailed, reason: - bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % reason ) - failedevent = TaskFailed(task, d) - event.fire(failedevent) - raise EventException("Function failed in task: %s" % reason, failedevent) + event.fire(TaskSucceeded(task, localdata), localdata) + except FuncFailed, message: + # Try to extract the optional logfile + try: + (msg, logfile) = message + except: + logfile = None + msg = message + bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % message ) + failedevent = TaskFailed(msg, logfile, task, d) + event.fire(failedevent, d) + raise EventException("Function failed in task: %s" % message, failedevent) # make stamp, or cause event and raise exception if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d): diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index d30d57d33b..2f1b8fa601 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py @@ -134,7 +134,18 @@ class Cache: self.data = data # Make sure __depends makes the depends_cache - self.getVar("__depends", virtualfn, True) + # If we're a virtual class we need to make sure all our depends are appended + # to the depends of fn. + depends = self.getVar("__depends", virtualfn, True) or [] + if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]: + self.depends_cache[fn]["__depends"] = depends + for dep in depends: + if dep not in self.depends_cache[fn]["__depends"]: + self.depends_cache[fn]["__depends"].append(dep) + + # Make sure BBCLASSEXTEND always makes the cache too + self.getVar('BBCLASSEXTEND', virtualfn, True) + self.depends_cache[virtualfn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn) def virtualfn2realfn(self, virtualfn): @@ -170,11 +181,8 @@ class Cache: bb.msg.debug(1, bb.msg.domain.Cache, "Parsing %s (full)" % fn) - bb_data, skipped = self.load_bbfile(fn, cfgData) - if isinstance(bb_data, dict): - return bb_data[cls] - - return bb_data + bb_data = self.load_bbfile(fn, cfgData) + return bb_data[cls] def loadData(self, fn, cfgData, cacheData): """ @@ -184,42 +192,39 @@ class Cache: to record the variables accessed. Return the cache status and whether the file was skipped when parsed """ + skipped = 0 + virtuals = 0 + if fn not in self.checked: self.cacheValidUpdate(fn) + if self.cacheValid(fn): - if "SKIPPED" in self.depends_cache[fn]: - return True, True - self.handle_data(fn, cacheData) multi = self.getVar('BBCLASSEXTEND', fn, True) - if multi: - for cls in multi.split(): - virtualfn = self.realfn2virtual(fn, cls) - # Pretend we're clean so getVar works - self.clean[virtualfn] = "" - self.handle_data(virtualfn, cacheData) - return True, False + for cls in (multi or "").split() + [""]: + virtualfn = self.realfn2virtual(fn, cls) + if self.depends_cache[virtualfn]["__SKIPPED"]: + skipped += 1 + bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn) + continue + self.handle_data(virtualfn, cacheData) + virtuals += 1 + return True, skipped, virtuals bb.msg.debug(1, bb.msg.domain.Cache, "Parsing %s" % fn) - bb_data, skipped = self.load_bbfile(fn, cfgData) - - if skipped: - if isinstance(bb_data, dict): - self.setData(fn, fn, bb_data[""]) - else: - self.setData(fn, fn, bb_data) - return False, skipped + bb_data = self.load_bbfile(fn, cfgData) - if isinstance(bb_data, dict): - for data in bb_data: - virtualfn = self.realfn2virtual(fn, data) - self.setData(virtualfn, fn, bb_data[data]) + for data in bb_data: + virtualfn = self.realfn2virtual(fn, data) + self.setData(virtualfn, fn, bb_data[data]) + if self.getVar("__SKIPPED", virtualfn, True): + skipped += 1 + bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn) + else: self.handle_data(virtualfn, cacheData) - return False, skipped + virtuals += 1 + return False, skipped, virtuals - self.setData(fn, fn, bb_data) - self.handle_data(fn, cacheData) - return False, skipped def cacheValid(self, fn): """ @@ -286,16 +291,13 @@ class Cache: if not fn in self.clean: self.clean[fn] = "" - return True + # Mark extended class data as clean too + multi = self.getVar('BBCLASSEXTEND', fn, True) + for cls in (multi or "").split(): + virtualfn = self.realfn2virtual(fn, cls) + self.clean[virtualfn] = "" - def skip(self, fn): - """ - Mark a fn as skipped - Called from the parser - """ - if not fn in self.depends_cache: - self.depends_cache[fn] = {} - self.depends_cache[fn]["SKIPPED"] = "1" + return True def remove(self, fn): """ @@ -462,10 +464,7 @@ class Cache: try: bb_data = parse.handle(bbfile, bb_data) # read .bb data os.chdir(oldpath) - return bb_data, False - except bb.parse.SkipPackage: - os.chdir(oldpath) - return bb_data, True + return bb_data except: os.chdir(oldpath) raise diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py new file mode 100644 index 0000000000..2bb5365c0c --- /dev/null +++ b/bitbake/lib/bb/command.py @@ -0,0 +1,271 @@ +""" +BitBake 'Command' module + +Provide an interface to interact with the bitbake server through 'commands' +""" + +# Copyright (C) 2006-2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +The bitbake server takes 'commands' from its UI/commandline. +Commands are either synchronous or asynchronous. +Async commands return data to the client in the form of events. +Sync commands must only return data through the function return value +and must not trigger events, directly or indirectly. +Commands are queued in a CommandQueue +""" + +import bb + +async_cmds = {} +sync_cmds = {} + +class Command: + """ + A queue of asynchronous commands for bitbake + """ + def __init__(self, cooker): + + self.cooker = cooker + self.cmds_sync = CommandsSync() + self.cmds_async = CommandsAsync() + + # FIXME Add lock for this + self.currentAsyncCommand = None + + for attr in CommandsSync.__dict__: + command = attr[:].lower() + method = getattr(CommandsSync, attr) + sync_cmds[command] = (method) + + for attr in CommandsAsync.__dict__: + command = attr[:].lower() + method = getattr(CommandsAsync, attr) + async_cmds[command] = (method) + + def runCommand(self, commandline): + try: + command = commandline.pop(0) + if command in CommandsSync.__dict__: + # Can run synchronous commands straight away + return getattr(CommandsSync, command)(self.cmds_sync, self, commandline) + if self.currentAsyncCommand is not None: + return "Busy (%s in progress)" % self.currentAsyncCommand[0] + if command not in CommandsAsync.__dict__: + return "No such command" + self.currentAsyncCommand = (command, commandline) + self.cooker.server.register_idle_function(self.cooker.runCommands, self.cooker) + return True + except: + import traceback + return traceback.format_exc() + + def runAsyncCommand(self): + try: + if self.currentAsyncCommand is not None: + (command, options) = self.currentAsyncCommand + commandmethod = getattr(CommandsAsync, command) + needcache = getattr( commandmethod, "needcache" ) + if needcache and self.cooker.cookerState != bb.cooker.cookerParsed: + self.cooker.updateCache() + return True + else: + commandmethod(self.cmds_async, self, options) + return False + else: + return False + except: + import traceback + self.finishAsyncCommand(traceback.format_exc()) + return False + + def finishAsyncCommand(self, error = None): + if error: + bb.event.fire(bb.command.CookerCommandFailed(error), self.cooker.configuration.event_data) + else: + bb.event.fire(bb.command.CookerCommandCompleted(), self.cooker.configuration.event_data) + self.currentAsyncCommand = None + + +class CommandsSync: + """ + A class of synchronous commands + These should run quickly so as not to hurt interactive performance. + These must not influence any running synchronous command. + """ + + def stateShutdown(self, command, params): + """ + Trigger cooker 'shutdown' mode + """ + command.cooker.cookerAction = bb.cooker.cookerShutdown + + def stateStop(self, command, params): + """ + Stop the cooker + """ + command.cooker.cookerAction = bb.cooker.cookerStop + + def getCmdLineAction(self, command, params): + """ + Get any command parsed from the commandline + """ + return command.cooker.commandlineAction + + def getVariable(self, command, params): + """ + Read the value of a variable from configuration.data + """ + varname = params[0] + expand = True + if len(params) > 1: + expand = params[1] + + return bb.data.getVar(varname, command.cooker.configuration.data, expand) + + def setVariable(self, command, params): + """ + Set the value of variable in configuration.data + """ + varname = params[0] + value = params[1] + bb.data.setVar(varname, value, command.cooker.configuration.data) + + +class CommandsAsync: + """ + A class of asynchronous commands + These functions communicate via generated events. + Any function that requires metadata parsing should be here. + """ + + def buildFile(self, command, params): + """ + Build a single specified .bb file + """ + bfile = params[0] + task = params[1] + + command.cooker.buildFile(bfile, task) + buildFile.needcache = False + + def buildTargets(self, command, params): + """ + Build a set of targets + """ + pkgs_to_build = params[0] + task = params[1] + + command.cooker.buildTargets(pkgs_to_build, task) + buildTargets.needcache = True + + def generateDepTreeEvent(self, command, params): + """ + Generate an event containing the dependency information + """ + pkgs_to_build = params[0] + task = params[1] + + command.cooker.generateDepTreeEvent(pkgs_to_build, task) + command.finishAsyncCommand() + generateDepTreeEvent.needcache = True + + def generateDotGraph(self, command, params): + """ + Dump dependency information to disk as .dot files + """ + pkgs_to_build = params[0] + task = params[1] + + command.cooker.generateDotGraphFiles(pkgs_to_build, task) + command.finishAsyncCommand() + generateDotGraph.needcache = True + + def showVersions(self, command, params): + """ + Show the currently selected versions + """ + command.cooker.showVersions() + command.finishAsyncCommand() + showVersions.needcache = True + + def showEnvironmentTarget(self, command, params): + """ + Print the environment of a target recipe + (needs the cache to work out which recipe to use) + """ + pkg = params[0] + + command.cooker.showEnvironment(None, pkg) + command.finishAsyncCommand() + showEnvironmentTarget.needcache = True + + def showEnvironment(self, command, params): + """ + Print the standard environment + or if specified the environment for a specified recipe + """ + bfile = params[0] + + command.cooker.showEnvironment(bfile) + command.finishAsyncCommand() + showEnvironment.needcache = False + + def parseFiles(self, command, params): + """ + Parse the .bb files + """ + command.cooker.updateCache() + command.finishAsyncCommand() + parseFiles.needcache = True + + def compareRevisions(self, command, params): + """ + Parse the .bb files + """ |
