diff options
Diffstat (limited to 'bitbake/lib/bb/build.py')
| -rw-r--r-- | bitbake/lib/bb/build.py | 488 |
1 files changed, 257 insertions, 231 deletions
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py index 18a75edca3..07bd35afcc 100644 --- a/bitbake/lib/bb/build.py +++ b/bitbake/lib/bb/build.py @@ -25,9 +25,20 @@ # #Based on functions from the base bb module, Copyright 2003 Holger Schurig +import os +import sys +import logging +import bb +import bb.msg +import bb.process +from contextlib import nested from bb import data, event, mkdirhier, utils -import bb, os, sys -import bb.utils + +bblogger = logging.getLogger('BitBake') +logger = logging.getLogger('BitBake.Build') + +NULL = open(os.devnull, 'r+') + # When we execute a python function we'd like certain things # in all namespaces, hence we add them to __builtins__ @@ -36,13 +47,22 @@ import bb.utils __builtins__['bb'] = bb __builtins__['os'] = os -# events class FuncFailed(Exception): - """ - Executed function failed - First parameter a message - Second paramter is a logfile (optional) - """ + def __init__(self, name = None, logfile = None): + self.logfile = logfile + self.name = name + if name: + self.msg = "Function '%s' failed" % name + else: + self.msg = "Function failed" + + def __str__(self): + if self.logfile and os.path.exists(self.logfile): + msg = ("%s (see %s for further information)" % + (self.msg, self.logfile)) + else: + msg = self.msg + return msg class TaskBase(event.Event): """Base class for task events""" @@ -69,38 +89,56 @@ class TaskSucceeded(TaskBase): class TaskFailed(TaskBase): """Task execution failed""" - def __init__(self, msg, logfile, t, d ): + + def __init__(self, task, logfile, metadata): self.logfile = logfile - self.msg = msg - TaskBase.__init__(self, t, d) + super(TaskFailed, self).__init__(task, metadata) class TaskInvalid(TaskBase): - """Invalid Task""" -# functions + def __init__(self, task, metadata): + super(TaskInvalid, self).__init__(task, metadata) + self._message = "No such task '%s'" % task + + +class LogTee(object): + def __init__(self, logger, outfile): + self.outfile = outfile + self.logger = logger + self.name = self.outfile.name + + def write(self, string): + self.logger.plain(string) + self.outfile.write(string) + + def __enter__(self): + self.outfile.__enter__() + return self + + def __exit__(self, *excinfo): + self.outfile.__exit__(*excinfo) + + def __repr__(self): + return '<LogTee {0}>'.format(self.name) + def exec_func(func, d, dirs = None): """Execute an BB 'function'""" body = data.getVar(func, d) if not body: - bb.warn("Function %s doesn't exist" % func) + if body is None: + logger.warn("Function %s doesn't exist", func) return flags = data.getVarFlags(func, d) - for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot', 'task']: - if not item in flags: - flags[item] = None - - ispython = flags['python'] - - cleandirs = flags['cleandirs'] + cleandirs = flags.get('cleandirs') if cleandirs: for cdir in data.expand(cleandirs, d).split(): - os.system("rm -rf %s" % cdir) + bb.utils.remove(cdir, True) if dirs is None: - dirs = flags['dirs'] + dirs = flags.get('dirs') if dirs: dirs = data.expand(dirs, d).split() @@ -110,277 +148,254 @@ def exec_func(func, d, dirs = None): adir = dirs[-1] else: adir = data.getVar('B', d, 1) + if not os.path.exists(adir): + adir = None - # Save current directory - try: - prevdir = os.getcwd() - except OSError: - prevdir = data.getVar('TOPDIR', d, True) - - # Setup scriptfile - t = data.getVar('T', d, 1) - if not t: - raise SystemExit("T variable not set, unable to build") - bb.utils.mkdirhier(t) - runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) - logfile = d.getVar("BB_LOGFILE", True) - - # Change to correct directory (if specified) - if adir and os.access(adir, os.F_OK): - os.chdir(adir) - - locks = [] - lockfiles = flags['lockfiles'] - if lockfiles: - for lock in data.expand(lockfiles, d).split(): - locks.append(bb.utils.lockfile(lock)) + ispython = flags.get('python') + if flags.get('fakeroot') and not flags.get('task'): + bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func) - try: - # Run the function - if ispython: - exec_func_python(func, d, runfile, logfile) - else: - exec_func_shell(func, d, runfile, logfile, flags) + lockflag = flags.get('lockfiles') + if lockflag: + lockfiles = [data.expand(f, d) for f in lockflag.split()] + else: + lockfiles = None - # Restore original directory - try: - os.chdir(prevdir) - except: - pass + tempdir = data.getVar('T', d, 1) + runfile = os.path.join(tempdir, 'run.{0}.{1}'.format(func, os.getpid())) - finally: + with bb.utils.fileslocked(lockfiles): + if ispython: + exec_func_python(func, d, runfile, cwd=adir) + else: + exec_func_shell(func, d, runfile, cwd=adir) - # Unlock any lockfiles - for lock in locks: - bb.utils.unlockfile(lock) +_functionfmt = """ +def {function}(d): +{body} -def exec_func_python(func, d, runfile, logfile): +{function}(d) +""" +logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") +def exec_func_python(func, d, runfile, cwd=None): """Execute a python BB 'function'""" - bbfile = bb.data.getVar('FILE', d, 1) - tmp = "def " + func + "(d):\n%s" % data.getVar(func, d) - tmp += '\n' + func + '(d)' + bbfile = d.getVar('FILE', True) + try: + olddir = os.getcwd() + except OSError: + olddir = None + code = _functionfmt.format(function=func, body=d.getVar(func, True)) + bb.utils.mkdirhier(os.path.dirname(runfile)) + with open(runfile, 'w') as script: + script.write(code) + + if cwd: + os.chdir(cwd) - f = open(runfile, "w") - f.write(tmp) - comp = utils.better_compile(tmp, func, bbfile) try: - utils.better_exec(comp, {"d": d}, tmp, bbfile) + comp = utils.better_compile(code, func, bbfile) + utils.better_exec(comp, {"d": d}, code, bbfile) except: - (t, value, tb) = sys.exc_info() - - if t in [bb.parse.SkipPackage, bb.build.FuncFailed]: + if sys.exc_info()[0] in (bb.parse.SkipPackage, bb.build.FuncFailed): raise - raise FuncFailed("Function %s failed" % func, logfile) + raise FuncFailed(func, None) + finally: + if olddir: + os.chdir(olddir) -def exec_func_shell(func, d, runfile, logfile, flags): - """Execute a shell BB 'function' Returns true if execution was successful. - - For this, it creates a bash shell script in the tmp dectory, writes the local - data into it and finally executes. The output of the shell will end in a log file and stdout. +def exec_func_shell(function, d, runfile, cwd=None): + """Execute a shell function from the metadata Note on directory behavior. The 'dirs' varflag should contain a list of the directories you need created prior to execution. The last item in the list is where we will chdir/cd to. """ - deps = flags['deps'] - check = flags['check'] - if check in globals(): - if globals()[check](func, deps): - return - - f = open(runfile, "w") - f.write("#!/bin/sh -e\n") - if bb.msg.debug_level['default'] > 0: f.write("set -x\n") - data.emit_func(func, f, d) - - f.write("cd %s\n" % os.getcwd()) - if func: f.write("%s\n" % func) - f.close() - os.chmod(runfile, 0775) - if not func: - raise FuncFailed("Function not specified for exec_func_shell") - - # execute function - if flags['fakeroot'] and not flags['task']: - bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func) - - lang_environment = "LC_ALL=C " - ret = os.system('%ssh -e %s' % (lang_environment, runfile)) + # Don't let the emitted shell script override PWD + d.delVarFlag('PWD', 'export') - if ret == 0: - return + with open(runfile, 'w') as script: + script.write('#!/bin/sh -e\n') + if logger.isEnabledFor(logging.DEBUG): + script.write("set -x\n") + data.emit_func(function, script, d) - raise FuncFailed("function %s failed" % func, logfile) + script.write("%s\n" % function) + os.fchmod(script.fileno(), 0775) + env = { + 'PATH': d.getVar('PATH', True), + 'LC_ALL': 'C', + } -def exec_task(fn, task, d): - """Execute an BB 'task' + cmd = runfile - The primary difference between executing a task versus executing - a function is that a task exists in the task digraph, and therefore - has dependencies amongst other tasks.""" + if logger.isEnabledFor(logging.DEBUG): + logfile = LogTee(logger, sys.stdout) + else: + logfile = sys.stdout - # Check whther this is a valid task + try: + bb.process.run(cmd, env=env, cwd=cwd, shell=False, stdin=NULL, + log=logfile) + except bb.process.CmdError: + logfn = d.getVar('BB_LOGFILE', True) + raise FuncFailed(function, logfn) + +def _task_data(fn, task, d): + localdata = data.createCopy(d) + localdata.setVar('BB_FILENAME', fn) + localdata.setVar('BB_CURRENTTASK', task[3:]) + localdata.setVar('OVERRIDES', 'task-%s:%s' % + (task[3:], d.getVar('OVERRIDES', False))) + localdata.finalize() + data.expandKeys(localdata) + return localdata + +def _exec_task(fn, task, d, quieterr): + """Execute a BB 'task' + + Execution of a task involves a bit more setup than executing a function, + running it with its own local metadata, and with some useful variables set. + """ if not data.getVarFlag(task, 'task', d): event.fire(TaskInvalid(task, d), d) - bb.msg.error(bb.msg.domain.Build, "No such task: %s" % task) + logger.error("No such task: %s" % task) return 1 - quieterr = False - if d.getVarFlag(task, "quieterrors") is not None: - quieterr = True + logger.debug(1, "Executing task %s", task) + + localdata = _task_data(fn, task, d) + tempdir = localdata.getVar('T', True) + if not tempdir: + bb.fatal("T variable not set, unable to build") + + bb.utils.mkdirhier(tempdir) + loglink = os.path.join(tempdir, 'log.{0}'.format(task)) + logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid())) + if loglink: + bb.utils.remove(loglink) - try: - bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % task) - old_overrides = data.getVar('OVERRIDES', d, 0) - localdata = data.createCopy(d) - data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata) - data.update_data(localdata) - data.expandKeys(localdata) - data.setVar('BB_FILENAME', fn, d) - data.setVar('BB_CURRENTTASK', task[3:], d) - event.fire(TaskStarted(task, localdata), localdata) - - # Setup logfiles - t = data.getVar('T', d, 1) - if not t: - raise SystemExit("T variable not set, unable to build") - bb.utils.mkdirhier(t) - loglink = "%s/log.%s" % (t, task) - logfile = "%s/log.%s.%s" % (t, task, str(os.getpid())) - d.setVar("BB_LOGFILE", logfile) - - # Even though the log file has not yet been opened, lets create the link - if loglink: - try: - os.remove(loglink) - except OSError as e: - pass - - try: - os.symlink(logfile, loglink) - except OSError as e: - pass - - # Handle logfiles - si = file('/dev/null', 'r') try: - so = file(logfile, 'w') - except OSError as e: - bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e) - pass - se = so - - # Dup the existing fds so we dont lose them - osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] - oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] - ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] - - # Replace those fds with our own - os.dup2(si.fileno(), osi[1]) - os.dup2(so.fileno(), oso[1]) - os.dup2(se.fileno(), ose[1]) - - # Since we've remapped stdout and stderr, its safe for log messages to be printed there now - # exec_func can nest so we have to save state - origstdout = bb.event.useStdout - bb.event.useStdout = True - - - prefuncs = (data.getVarFlag(task, 'prefuncs', localdata) or "").split() - for func in prefuncs: - exec_func(func, localdata) - exec_func(task, localdata) - postfuncs = (data.getVarFlag(task, 'postfuncs', localdata) or "").split() - for func in postfuncs: - exec_func(func, localdata) + os.symlink(logfn, loglink) + except OSError: + pass - event.fire(TaskSucceeded(task, localdata), localdata) + prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) + postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) - # make stamp, or cause event and raise exception - if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d): - make_stamp(task, d) + # Handle logfiles + si = file('/dev/null', 'r') + try: + logfile = file(logfn, 'w') + except OSError: + logger.exception("Opening log file '%s'", logfn) + pass - except FuncFailed as message: - # Try to extract the optional logfile - try: - (msg, logfile) = message - except: - logfile = None - msg = message - if not quieterr: - bb.msg.error(bb.msg.domain.Build, "Task failed: %s" % message ) - failedevent = TaskFailed(msg, logfile, task, d) - event.fire(failedevent, d) - return 1 + # Dup the existing fds so we dont lose them + osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] + oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] + ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] - except Exception: - from traceback import format_exc + # Replace those fds with our own + os.dup2(si.fileno(), osi[1]) + os.dup2(logfile.fileno(), oso[1]) + os.dup2(logfile.fileno(), ose[1]) + + # Ensure python logging goes to the logfile + handler = logging.StreamHandler(logfile) + handler.setFormatter(logformatter) + bblogger.addHandler(handler) + + localdata.setVar('BB_LOGFILE', logfn) + + event.fire(TaskStarted(task, localdata), localdata) + try: + for func in (prefuncs or '').split(): + exec_func(func, localdata) + exec_func(task, localdata) + for func in (postfuncs or '').split(): + exec_func(func, localdata) + except FuncFailed as exc: if not quieterr: - bb.msg.error(bb.msg.domain.Build, "Build of %s failed" % (task)) - bb.msg.error(bb.msg.domain.Build, format_exc()) - failedevent = TaskFailed("Task Failed", None, task, d) - event.fire(failedevent, d) + logger.error(str(exc)) + event.fire(TaskFailed(exc.name, logfn, localdata), localdata) return 1 finally: sys.stdout.flush() sys.stderr.flush() - bb.event.useStdout = origstdout + bblogger.removeHandler(handler) # Restore the backup fds os.dup2(osi[0], osi[1]) os.dup2(oso[0], oso[1]) os.dup2(ose[0], ose[1]) - # Close our logs - si.close() - so.close() - se.close() - - if logfile and os.path.exists(logfile) and os.path.getsize(logfile) == 0: - bb.msg.debug(2, bb.msg.domain.Build, "Zero size logfile %s, removing" % logfile) - os.remove(logfile) - try: - os.remove(loglink) - except OSError as e: - pass - # Close the backup fds os.close(osi[0]) os.close(oso[0]) os.close(ose[0]) + si.close() + + logfile.close() + if os.path.exists(logfn) and os.path.getsize(logfn) == 0: + logger.debug(2, "Zero size logfn %s, removing", logfn) + bb.utils.remove(logfn) + bb.utils.remove(loglink) + event.fire(TaskSucceeded(task, localdata), localdata) + + if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'): + make_stamp(task, localdata) return 0 -def extract_stamp(d, fn): - """ - Extracts stamp format which is either a data dictionary (fn unset) - or a dataCache entry (fn set). - """ - if fn: - return d.stamp[fn] - return data.getVar('STAMP', d, 1) +def exec_task(fn, task, d): + try: + quieterr = False + if d.getVarFlag(task, "quieterrors") is not None: + quieterr = True + + return _exec_task(fn, task, d, quieterr) + except Exception: + from traceback import format_exc + if not quieterr: + logger.error("Build of %s failed" % (task)) + logger.error(format_exc()) + failedevent = TaskFailed("Task Failed", None, task, d) + event.fire(failedevent, d) + return 1 -def stamp_internal(task, d, file_name): +def stamp_internal(taskname, d, file_name): """ Internal stamp helper function - Removes any stamp for the given task Makes sure the stamp directory exists Returns the stamp path+filename + + In the bitbake core, d can be a CacheData and file_name will be set. + When called in task context, d will be a data store, file_name will not be set """ - stamp = extract_stamp(d, file_name) + taskflagname = taskname + if taskname.endswith("_setscene") and taskname != "do_setscene": + taskflagname = taskname.replace("_setscene", "") + + if file_name: + stamp = d.stamp[file_name] + extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" + else: + stamp = d.getVar('STAMP', True) + file_name = d.getVar('BB_FILENAME', True) + extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or "" + if not stamp: return - stamp = "%s.%s" % (stamp, task) + + stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo) + bb.utils.mkdirhier(os.path.dirname(stamp)) - # Remove the file and recreate to force timestamp - # change on broken NFS filesystems - if os.access(stamp, os.F_OK): - os.remove(stamp) + return stamp def make_stamp(task, d, file_name = None): @@ -389,7 +404,10 @@ def make_stamp(task, d, file_name = None): (d can be a data dict or dataCache) """ stamp = stamp_internal(task, d, file_name) + # Remove the file and recreate to force timestamp + # change on broken NFS filesystems if stamp: + bb.utils.remove(stamp) f = open(stamp, "w") f.close() @@ -398,7 +416,15 @@ def del_stamp(task, d, file_name = None): Removes a stamp for a given task (d can be a data dict or dataCache) """ - stamp_internal(task, d, file_name) + stamp = stamp_internal(task, d, file_name) + bb.utils.remove(stamp) + +def stampfile(taskname, d, file_name = None): + """ + Return the stamp for a given task + (d can be a data dict or dataCache) + """ + return stamp_internal(taskname, d, file_name) def add_tasks(tasklist, d): task_deps = data.getVar('_task_deps', d) |