diff options
Diffstat (limited to 'bitbake/lib/bb/utils.py')
| -rw-r--r-- | bitbake/lib/bb/utils.py | 177 |
1 files changed, 105 insertions, 72 deletions
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index f468fafc12..6373912d88 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py @@ -21,10 +21,14 @@ BitBake Utility Functions import re, fcntl, os, string, stat, shutil, time import sys -import bb import errno +import logging +import bb import bb.msg from commands import getstatusoutput +from contextlib import contextmanager + +logger = logging.getLogger("BitBake.Util") # Version comparison separators = ".-" @@ -90,7 +94,7 @@ def vercmp(ta, tb): (ea, va, ra) = ta (eb, vb, rb) = tb - r = int(ea)-int(eb) + r = int(ea or 0) - int(eb or 0) if (r == 0): r = vercmp_part(va, vb) if (r == 0): @@ -191,10 +195,10 @@ def vercmp_string(val1, val2): val2 = val2[0].split('.') # add back decimal point so that .03 does not become "3" ! - for x in range(1, len(val1)): + for x in xrange(1, len(val1)): if val1[x][0] == '0' : val1[x] = '.' + val1[x] - for x in range(1, len(val2)): + for x in xrange(1, len(val2)): if val2[x][0] == '0' : val2[x] = '.' + val2[x] @@ -211,10 +215,10 @@ def vercmp_string(val1, val2): val2[-1] += '_' + val2_prepart # The above code will extend version numbers out so they # have the same number of digits. - for x in range(0, len(val1)): + for x in xrange(0, len(val1)): cmp1 = relparse(val1[x]) cmp2 = relparse(val2[x]) - for y in range(0, 3): + for y in xrange(0, 3): myret = cmp1[y] - cmp2[y] if myret != 0: __vercmp_cache__[valkey] = myret @@ -287,17 +291,6 @@ def join_deps(deps): result.append(dep) return ", ".join(result) -def extend_deps(dest, src): - """ - Extend the results from explode_dep_versions by appending all of the items - in the second list, avoiding duplicates. - """ - for dep in src: - if dep not in dest: - dest[dep] = src[dep] - elif dest[dep] != src[dep]: - dest[dep] = src[dep] - def _print_trace(body, line): """ Print the Environment of a Text Body @@ -305,12 +298,11 @@ def _print_trace(body, line): # print the environment of the method min_line = max(1, line-4) max_line = min(line + 4, len(body)) - for i in range(min_line, max_line + 1): + for i in xrange(min_line, max_line + 1): if line == i: - bb.msg.error(bb.msg.domain.Util, " *** %.4d:%s" % (i, body[i-1]) ) + logger.error(' *** %.4d:%s', i, body[i-1]) else: - bb.msg.error(bb.msg.domain.Util, " %.4d:%s" % (i, body[i-1]) ) - + logger.error(' %.4d:%s', i, body[i-1]) def better_compile(text, file, realfile, mode = "exec"): """ @@ -322,62 +314,65 @@ def better_compile(text, file, realfile, mode = "exec"): except Exception as e: # split the text into lines again body = text.split('\n') - bb.msg.error(bb.msg.domain.Util, "Error in compiling python function in: %s" % (realfile)) - bb.msg.error(bb.msg.domain.Util, str(e)) + logger.error("Error in compiling python function in %s", realfile) + logger.error(str(e)) if e.lineno: - bb.msg.error(bb.msg.domain.Util, "The lines leading to this error were:") - bb.msg.error(bb.msg.domain.Util, "\t%d:%s:'%s'" % (e.lineno, e.__class__.__name__, body[e.lineno-1])) + logger.error("The lines leading to this error were:") + logger.error("\t%d:%s:'%s'", e.lineno, e.__class__.__name__, body[e.lineno-1]) _print_trace(body, e.lineno) else: - bb.msg.error(bb.msg.domain.Util, "The function causing this error was:") + logger.error("The function causing this error was:") for line in body: - bb.msg.error(bb.msg.domain.Util, line) + logger.error(line) + raise -def better_exec(code, context, text, realfile): +def better_exec(code, context, text, realfile = "<code>"): """ Similiar to better_compile, better_exec will print the lines that are responsible for the error. """ import bb.parse + if not hasattr(code, "co_filename"): + code = better_compile(code, realfile, realfile) try: exec(code, _context, context) - except: + except Exception: (t, value, tb) = sys.exc_info() if t in [bb.parse.SkipPackage, bb.build.FuncFailed]: raise - # print the Header of the Error Message - bb.msg.error(bb.msg.domain.Util, "There was an error when executing a python function in: %s" % realfile) - bb.msg.error(bb.msg.domain.Util, "Exception:%s Message:%s" % (t, value)) + import traceback + exception = traceback.format_exception_only(t, value) + logger.error('Error executing a python function in %s:\n%s', + realfile, ''.join(exception)) # Strip 'us' from the stack (better_exec call) tb = tb.tb_next - import traceback textarray = text.split('\n') linefailed = traceback.tb_lineno(tb) tbextract = traceback.extract_tb(tb) tbformat = "\n".join(traceback.format_list(tbextract)) - bb.msg.error(bb.msg.domain.Util, "The stack trace of python calls that resulted in thie exception/failure was:") + logger.error("The stack trace of python calls that resulted in this exception/failure was:") for line in tbformat.split('\n'): - bb.msg.error(bb.msg.domain.Util, line) + logger.error(line) - bb.msg.error(bb.msg.domain.Util, "The code that was being executed was:") + logger.error("The code that was being executed was:") _print_trace(textarray, linefailed) - bb.msg.error(bb.msg.domain.Util, "(file: '%s', lineno: %s, function: %s)" % (tbextract[0][0], tbextract[0][1], tbextract[0][2])) + logger.error("(file: '%s', lineno: %s, function: %s)", tbextract[0][0], tbextract[0][1], tbextract[0][2]) - # See if this is a function we constructed and has calls back into other functions in + # See if this is a function we constructed and has calls back into other functions in # "text". If so, try and improve the context of the error by diving down the trace level = 0 nexttb = tb.tb_next while nexttb is not None: if tbextract[level][0] == tbextract[level+1][0] and tbextract[level+1][2] == tbextract[level][0]: _print_trace(textarray, tbextract[level+1][1]) - bb.msg.error(bb.msg.domain.Util, "(file: '%s', lineno: %s, function: %s)" % (tbextract[level+1][0], tbextract[level+1][1], tbextract[level+1][2])) + logger.error("(file: '%s', lineno: %s, function: %s)", tbextract[level+1][0], tbextract[level+1][1], tbextract[level+1][2]) else: break nexttb = tb.tb_next @@ -391,16 +386,37 @@ def simple_exec(code, context): def better_eval(source, locals): return eval(source, _context, locals) -def lockfile(name): +@contextmanager +def fileslocked(files): + """Context manager for locking and unlocking file locks.""" + locks = [] + if files: + for lockfile in files: + locks.append(bb.utils.lockfile(lockfile)) + + yield + + for lock in locks: + bb.utils.unlockfile(lock) + +def lockfile(name, shared=False): """ Use the file fn as a lock file, return when the lock has been acquired. Returns a variable to pass to unlockfile(). """ path = os.path.dirname(name) if not os.path.isdir(path): - bb.msg.error(bb.msg.domain.Util, "Error, lockfile path does not exist!: %s" % path) + logger.error("Lockfile destination directory '%s' does not exist", path) + sys.exit(1) + + if not os.access(path, os.W_OK): + logger.error("Error, lockfile path is not writable!: %s" % path) sys.exit(1) + op = fcntl.LOCK_EX + if shared: + op = fcntl.LOCK_SH + while True: # If we leave the lockfiles lying around there is no problem # but we should clean up after ourselves. This gives potential @@ -413,25 +429,31 @@ def lockfile(name): # lock is the most likely to win it. try: - lf = open(name, "a + ") - fcntl.flock(lf.fileno(), fcntl.LOCK_EX) - statinfo = os.fstat(lf.fileno()) + lf = open(name, 'a+') + fileno = lf.fileno() + fcntl.flock(fileno, op) + statinfo = os.fstat(fileno) if os.path.exists(lf.name): statinfo2 = os.stat(lf.name) if statinfo.st_ino == statinfo2.st_ino: return lf - # File no longer exists or changed, retry - lf.close - except Exception as e: + lf.close() + except Exception: continue def unlockfile(lf): """ Unlock a file locked using lockfile() """ - os.unlink(lf.name) + try: + # If we had a shared lock, we need to promote to exclusive before + # removing the lockfile. Attempt this, ignore failures. + fcntl.flock(lf.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB) + os.unlink(lf.name) + except (IOError, OSError): + pass fcntl.flock(lf.fileno(), fcntl.LOCK_UN) - lf.close + lf.close() def md5_file(filename): """ @@ -465,9 +487,9 @@ def sha256_file(filename): s.update(line) return s.hexdigest() -# Variables which are preserved from the original environment *and* exported -# into our worker context -def preserved_envvars_export_list(): +def preserved_envvars_exported(): + """Variables which are taken from the environment and placed in and exported + from the metadata""" return [ 'BB_TASKHASH', 'HOME', @@ -480,9 +502,9 @@ def preserved_envvars_export_list(): 'USERNAME', ] -# Variables which are preserved from the original environment *and* exported -# into our worker context for interactive tasks (e.g. requiring X) -def preserved_envvars_export_interactive_list(): +def preserved_envvars_exported_interactive(): + """Variables which are taken from the environment and placed in and exported + from the metadata, for interactive tasks""" return [ 'COLORTERM', 'DBUS_SESSION_BUS_ADDRESS', @@ -500,8 +522,8 @@ def preserved_envvars_export_interactive_list(): 'XDG_SESSION_COOKIE', ] -# Variables which are preserved from the original environment into the datastore -def preserved_envvars_list(): +def preserved_envvars(): + """Variables which are taken from the environment and placed in the metadata""" v = [ 'BBPATH', 'BB_PRESERVE_ENV', @@ -510,7 +532,7 @@ def preserved_envvars_list(): 'LANG', '_', ] - return v + preserved_envvars_export_list() + preserved_envvars_export_interactive_list() + return v + preserved_envvars_exported() + preserved_envvars_exported_interactive() def filter_environment(good_vars): """ @@ -528,12 +550,12 @@ def filter_environment(good_vars): del os.environ[key] if len(removed_vars): - bb.msg.debug(1, bb.msg.domain.Util, "Removed the following variables from the environment: %s" % (", ".join(removed_vars))) + logger.debug(1, "Removed the following variables from the environment: %s", ", ".join(removed_vars)) return removed_vars -def create_intereactive_env(d): - for k in preserved_envvars_export_interactive_list(): +def create_interactive_env(d): + for k in preserved_envvars_exported_interactive(): os.setenv(k, bb.data.getVar(k, d, True)) def clean_environment(): @@ -545,7 +567,7 @@ def clean_environment(): if 'BB_ENV_WHITELIST' in os.environ: good_vars = os.environ['BB_ENV_WHITELIST'].split() else: - good_vars = preserved_envvars_list() + good_vars = preserved_envvars() if 'BB_ENV_EXTRAWHITE' in os.environ: good_vars.extend(os.environ['BB_ENV_EXTRAWHITE'].split()) filter_environment(good_vars) @@ -568,6 +590,20 @@ def build_environment(d): if export: os.environ[var] = bb.data.getVar(var, d, True) or "" +def remove(path, recurse=False): + """Equivalent to rm -f or rm -rf""" + if not path: + return + import os, errno, shutil, glob + for name in glob.glob(path): + try: + os.unlink(name) + except OSError as exc: + if recurse and exc.errno == errno.EISDIR: + shutil.rmtree(name) + elif exc.errno != errno.ENOENT: + raise + def prunedir(topdir): # Delete everything reachable from the directory named in 'topdir'. # CAUTION: This is dangerous! @@ -593,15 +629,13 @@ def prune_suffix(var, suffixes, d): return var.replace(suffix, "") return var -def mkdirhier(dir): +def mkdirhier(directory): """Create a directory like 'mkdir -p', but does not complain if directory already exists like os.makedirs """ - bb.msg.debug(3, bb.msg.domain.Util, "mkdirhier(%s)" % dir) try: - os.makedirs(dir) - bb.msg.debug(2, bb.msg.domain.Util, "created " + dir) + os.makedirs(directory) except OSError as e: if e.errno != errno.EEXIST: raise e @@ -787,13 +821,12 @@ def init_logger(logger, verbose, debug, debug_domains): Set verbosity and debug levels in the logger """ - if verbose: - logger.set_verbose(True) - if debug: - logger.set_debug_level(debug) + bb.msg.set_debug_level(debug) + elif verbose: + bb.msg.set_verbose(True) else: - logger.set_debug_level(0) + bb.msg.set_debug_level(0) if debug_domains: - logger.set_debug_domains(debug_domains) + bb.msg.set_debug_domains(debug_domains) |