From 951cbf3f65f347c7a7bbcae193218f9187a15fbf Mon Sep 17 00:00:00 2001 From: Khem Raj Date: Wed, 23 Feb 2011 10:48:51 -0800 Subject: bitbake: Remove in-tree version Bitbake should be used by checking it out from its own repo Signed-off-by: Khem Raj --- bitbake/lib/bb/COW.py | 323 --- bitbake/lib/bb/__init__.py | 139 -- bitbake/lib/bb/build.py | 472 ---- bitbake/lib/bb/cache.py | 632 ----- bitbake/lib/bb/codeparser.py | 336 --- bitbake/lib/bb/command.py | 271 --- bitbake/lib/bb/cooker.py | 1078 --------- bitbake/lib/bb/daemonize.py | 190 -- bitbake/lib/bb/data.py | 338 --- bitbake/lib/bb/data_smart.py | 428 ---- bitbake/lib/bb/event.py | 386 --- bitbake/lib/bb/fetch/__init__.py | 836 ------- bitbake/lib/bb/fetch/bzr.py | 148 -- bitbake/lib/bb/fetch/cvs.py | 172 -- bitbake/lib/bb/fetch/git.py | 339 --- bitbake/lib/bb/fetch/hg.py | 180 -- bitbake/lib/bb/fetch/local.py | 73 - bitbake/lib/bb/fetch/osc.py | 143 -- bitbake/lib/bb/fetch/perforce.py | 206 -- bitbake/lib/bb/fetch/repo.py | 98 - bitbake/lib/bb/fetch/ssh.py | 118 - bitbake/lib/bb/fetch/svk.py | 104 - bitbake/lib/bb/fetch/svn.py | 204 -- bitbake/lib/bb/fetch/wget.py | 93 - bitbake/lib/bb/fetch2/__init__.py | 1074 --------- bitbake/lib/bb/fetch2/bzr.py | 141 -- bitbake/lib/bb/fetch2/cvs.py | 181 -- bitbake/lib/bb/fetch2/git.py | 242 -- bitbake/lib/bb/fetch2/hg.py | 174 -- bitbake/lib/bb/fetch2/local.py | 80 - bitbake/lib/bb/fetch2/osc.py | 135 -- bitbake/lib/bb/fetch2/perforce.py | 196 -- bitbake/lib/bb/fetch2/repo.py | 98 - bitbake/lib/bb/fetch2/ssh.py | 120 - bitbake/lib/bb/fetch2/svk.py | 97 - bitbake/lib/bb/fetch2/svn.py | 180 -- bitbake/lib/bb/fetch2/wget.py | 91 - bitbake/lib/bb/methodpool.py | 84 - bitbake/lib/bb/msg.py | 200 -- bitbake/lib/bb/parse/__init__.py | 123 - bitbake/lib/bb/parse/ast.py | 446 ---- bitbake/lib/bb/parse/parse_py/BBHandler.py | 254 -- bitbake/lib/bb/parse/parse_py/ConfHandler.py | 139 -- bitbake/lib/bb/parse/parse_py/__init__.py | 33 - bitbake/lib/bb/persist_data.py | 194 -- bitbake/lib/bb/process.py | 109 - bitbake/lib/bb/providers.py | 330 --- bitbake/lib/bb/pysh/__init__.py | 0 bitbake/lib/bb/pysh/builtin.py | 710 ------ bitbake/lib/bb/pysh/interp.py | 1367 ----------- bitbake/lib/bb/pysh/lsprof.py | 116 - bitbake/lib/bb/pysh/pysh.py | 167 -- bitbake/lib/bb/pysh/pyshlex.py | 888 ------- bitbake/lib/bb/pysh/pyshyacc.py | 779 ------ bitbake/lib/bb/pysh/sherrors.py | 41 - bitbake/lib/bb/pysh/subprocess_fix.py | 77 - bitbake/lib/bb/runqueue.py | 1663 ------------- bitbake/lib/bb/server/__init__.py | 0 bitbake/lib/bb/server/none.py | 195 -- bitbake/lib/bb/server/xmlrpc.py | 260 -- bitbake/lib/bb/shell.py | 820 ------- bitbake/lib/bb/siggen.py | 298 --- bitbake/lib/bb/taskdata.py | 586 ----- bitbake/lib/bb/ui/__init__.py | 17 - bitbake/lib/bb/ui/crumbs/__init__.py | 17 - bitbake/lib/bb/ui/crumbs/buildmanager.py | 455 ---- bitbake/lib/bb/ui/crumbs/progress.py | 17 - bitbake/lib/bb/ui/crumbs/puccho.glade | 606 ----- bitbake/lib/bb/ui/crumbs/runningbuild.py | 311 --- bitbake/lib/bb/ui/depexp.py | 307 --- bitbake/lib/bb/ui/goggle.py | 110 - bitbake/lib/bb/ui/knotty.py | 248 -- bitbake/lib/bb/ui/ncurses.py | 352 --- bitbake/lib/bb/ui/puccho.py | 425 ---- bitbake/lib/bb/ui/uievent.py | 127 - bitbake/lib/bb/ui/uihelper.py | 42 - bitbake/lib/bb/utils.py | 845 ------- bitbake/lib/codegen.py | 570 ----- bitbake/lib/ply/__init__.py | 4 - bitbake/lib/ply/lex.py | 1058 --------- bitbake/lib/ply/yacc.py | 3276 -------------------------- bitbake/lib/progressbar.py | 384 --- 82 files changed, 29166 deletions(-) delete mode 100644 bitbake/lib/bb/COW.py delete mode 100644 bitbake/lib/bb/__init__.py delete mode 100644 bitbake/lib/bb/build.py delete mode 100644 bitbake/lib/bb/cache.py delete mode 100644 bitbake/lib/bb/codeparser.py delete mode 100644 bitbake/lib/bb/command.py delete mode 100644 bitbake/lib/bb/cooker.py delete mode 100644 bitbake/lib/bb/daemonize.py delete mode 100644 bitbake/lib/bb/data.py delete mode 100644 bitbake/lib/bb/data_smart.py delete mode 100644 bitbake/lib/bb/event.py delete mode 100644 bitbake/lib/bb/fetch/__init__.py delete mode 100644 bitbake/lib/bb/fetch/bzr.py delete mode 100644 bitbake/lib/bb/fetch/cvs.py delete mode 100644 bitbake/lib/bb/fetch/git.py delete mode 100644 bitbake/lib/bb/fetch/hg.py delete mode 100644 bitbake/lib/bb/fetch/local.py delete mode 100644 bitbake/lib/bb/fetch/osc.py delete mode 100644 bitbake/lib/bb/fetch/perforce.py delete mode 100644 bitbake/lib/bb/fetch/repo.py delete mode 100644 bitbake/lib/bb/fetch/ssh.py delete mode 100644 bitbake/lib/bb/fetch/svk.py delete mode 100644 bitbake/lib/bb/fetch/svn.py delete mode 100644 bitbake/lib/bb/fetch/wget.py delete mode 100644 bitbake/lib/bb/fetch2/__init__.py delete mode 100644 bitbake/lib/bb/fetch2/bzr.py delete mode 100644 bitbake/lib/bb/fetch2/cvs.py delete mode 100644 bitbake/lib/bb/fetch2/git.py delete mode 100644 bitbake/lib/bb/fetch2/hg.py delete mode 100644 bitbake/lib/bb/fetch2/local.py delete mode 100644 bitbake/lib/bb/fetch2/osc.py delete mode 100644 bitbake/lib/bb/fetch2/perforce.py delete mode 100644 bitbake/lib/bb/fetch2/repo.py delete mode 100644 bitbake/lib/bb/fetch2/ssh.py delete mode 100644 bitbake/lib/bb/fetch2/svk.py delete mode 100644 bitbake/lib/bb/fetch2/svn.py delete mode 100644 bitbake/lib/bb/fetch2/wget.py delete mode 100644 bitbake/lib/bb/methodpool.py delete mode 100644 bitbake/lib/bb/msg.py delete mode 100644 bitbake/lib/bb/parse/__init__.py delete mode 100644 bitbake/lib/bb/parse/ast.py delete mode 100644 bitbake/lib/bb/parse/parse_py/BBHandler.py delete mode 100644 bitbake/lib/bb/parse/parse_py/ConfHandler.py delete mode 100644 bitbake/lib/bb/parse/parse_py/__init__.py delete mode 100644 bitbake/lib/bb/persist_data.py delete mode 100644 bitbake/lib/bb/process.py delete mode 100644 bitbake/lib/bb/providers.py delete mode 100644 bitbake/lib/bb/pysh/__init__.py delete mode 100644 bitbake/lib/bb/pysh/builtin.py delete mode 100644 bitbake/lib/bb/pysh/interp.py delete mode 100644 bitbake/lib/bb/pysh/lsprof.py delete mode 100644 bitbake/lib/bb/pysh/pysh.py delete mode 100644 bitbake/lib/bb/pysh/pyshlex.py delete mode 100644 bitbake/lib/bb/pysh/pyshyacc.py delete mode 100644 bitbake/lib/bb/pysh/sherrors.py delete mode 100644 bitbake/lib/bb/pysh/subprocess_fix.py delete mode 100644 bitbake/lib/bb/runqueue.py delete mode 100644 bitbake/lib/bb/server/__init__.py delete mode 100644 bitbake/lib/bb/server/none.py delete mode 100644 bitbake/lib/bb/server/xmlrpc.py delete mode 100644 bitbake/lib/bb/shell.py delete mode 100644 bitbake/lib/bb/siggen.py delete mode 100644 bitbake/lib/bb/taskdata.py delete mode 100644 bitbake/lib/bb/ui/__init__.py delete mode 100644 bitbake/lib/bb/ui/crumbs/__init__.py delete mode 100644 bitbake/lib/bb/ui/crumbs/buildmanager.py delete mode 100644 bitbake/lib/bb/ui/crumbs/progress.py delete mode 100644 bitbake/lib/bb/ui/crumbs/puccho.glade delete mode 100644 bitbake/lib/bb/ui/crumbs/runningbuild.py delete mode 100644 bitbake/lib/bb/ui/depexp.py delete mode 100644 bitbake/lib/bb/ui/goggle.py delete mode 100644 bitbake/lib/bb/ui/knotty.py delete mode 100644 bitbake/lib/bb/ui/ncurses.py delete mode 100644 bitbake/lib/bb/ui/puccho.py delete mode 100644 bitbake/lib/bb/ui/uievent.py delete mode 100644 bitbake/lib/bb/ui/uihelper.py delete mode 100644 bitbake/lib/bb/utils.py delete mode 100644 bitbake/lib/codegen.py delete mode 100644 bitbake/lib/ply/__init__.py delete mode 100644 bitbake/lib/ply/lex.py delete mode 100644 bitbake/lib/ply/yacc.py delete mode 100644 bitbake/lib/progressbar.py (limited to 'bitbake/lib') diff --git a/bitbake/lib/bb/COW.py b/bitbake/lib/bb/COW.py deleted file mode 100644 index 6917ec378a..0000000000 --- a/bitbake/lib/bb/COW.py +++ /dev/null @@ -1,323 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# This is a copy on write dictionary and set which abuses classes to try and be nice and fast. -# -# Copyright (C) 2006 Tim Amsell -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -#Please Note: -# Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW. -# Assign a file to __warn__ to get warnings about slow operations. -# - -from __future__ import print_function -import copy -import types -ImmutableTypes = ( - types.NoneType, - bool, - complex, - float, - int, - long, - tuple, - frozenset, - basestring -) - -MUTABLE = "__mutable__" - -class COWMeta(type): - pass - -class COWDictMeta(COWMeta): - __warn__ = False - __hasmutable__ = False - __marker__ = tuple() - - def __str__(cls): - # FIXME: I have magic numbers! - return "" % (cls.__count__, len(cls.__dict__) - 3) - __repr__ = __str__ - - def cow(cls): - class C(cls): - __count__ = cls.__count__ + 1 - return C - copy = cow - __call__ = cow - - def __setitem__(cls, key, value): - if not isinstance(value, ImmutableTypes): - if not isinstance(value, COWMeta): - cls.__hasmutable__ = True - key += MUTABLE - setattr(cls, key, value) - - def __getmutable__(cls, key, readonly=False): - nkey = key + MUTABLE - try: - return cls.__dict__[nkey] - except KeyError: - pass - - value = getattr(cls, nkey) - if readonly: - return value - - if not cls.__warn__ is False and not isinstance(value, COWMeta): - print("Warning: Doing a copy because %s is a mutable type." % key, file=cls.__warn__) - try: - value = value.copy() - except AttributeError as e: - value = copy.copy(value) - setattr(cls, nkey, value) - return value - - __getmarker__ = [] - def __getreadonly__(cls, key, default=__getmarker__): - """\ - Get a value (even if mutable) which you promise not to change. - """ - return cls.__getitem__(key, default, True) - - def __getitem__(cls, key, default=__getmarker__, readonly=False): - try: - try: - value = getattr(cls, key) - except AttributeError: - value = cls.__getmutable__(key, readonly) - - # This is for values which have been deleted - if value is cls.__marker__: - raise AttributeError("key %s does not exist." % key) - - return value - except AttributeError as e: - if not default is cls.__getmarker__: - return default - - raise KeyError(str(e)) - - def __delitem__(cls, key): - cls.__setitem__(key, cls.__marker__) - - def __revertitem__(cls, key): - if not cls.__dict__.has_key(key): - key += MUTABLE - delattr(cls, key) - - def __contains__(cls, key): - return cls.has_key(key) - - def has_key(cls, key): - value = cls.__getreadonly__(key, cls.__marker__) - if value is cls.__marker__: - return False - return True - - def iter(cls, type, readonly=False): - for key in dir(cls): - if key.startswith("__"): - continue - - if key.endswith(MUTABLE): - key = key[:-len(MUTABLE)] - - if type == "keys": - yield key - - try: - if readonly: - value = cls.__getreadonly__(key) - else: - value = cls[key] - except KeyError: - continue - - if type == "values": - yield value - if type == "items": - yield (key, value) - raise StopIteration() - - def iterkeys(cls): - return cls.iter("keys") - def itervalues(cls, readonly=False): - if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: - print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) - return cls.iter("values", readonly) - def iteritems(cls, readonly=False): - if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: - print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) - return cls.iter("items", readonly) - -class COWSetMeta(COWDictMeta): - def __str__(cls): - # FIXME: I have magic numbers! - return "" % (cls.__count__, len(cls.__dict__) -3) - __repr__ = __str__ - - def cow(cls): - class C(cls): - __count__ = cls.__count__ + 1 - return C - - def add(cls, value): - COWDictMeta.__setitem__(cls, repr(hash(value)), value) - - def remove(cls, value): - COWDictMeta.__delitem__(cls, repr(hash(value))) - - def __in__(cls, value): - return COWDictMeta.has_key(repr(hash(value))) - - def iterkeys(cls): - raise TypeError("sets don't have keys") - - def iteritems(cls): - raise TypeError("sets don't have 'items'") - -# These are the actual classes you use! -class COWDictBase(object): - __metaclass__ = COWDictMeta - __count__ = 0 - -class COWSetBase(object): - __metaclass__ = COWSetMeta - __count__ = 0 - -if __name__ == "__main__": - import sys - COWDictBase.__warn__ = sys.stderr - a = COWDictBase() - print("a", a) - - a['a'] = 'a' - a['b'] = 'b' - a['dict'] = {} - - b = a.copy() - print("b", b) - b['c'] = 'b' - - print() - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(): - print(x) - print() - - b['dict']['a'] = 'b' - b['a'] = 'c' - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(): - print(x) - print() - - try: - b['dict2'] - except KeyError as e: - print("Okay!") - - a['set'] = COWSetBase() - a['set'].add("o1") - a['set'].add("o1") - a['set'].add("o2") - - print("a", a) - for x in a['set'].itervalues(): - print(x) - print("--") - print("b", b) - for x in b['set'].itervalues(): - print(x) - print() - - b['set'].add('o3') - - print("a", a) - for x in a['set'].itervalues(): - print(x) - print("--") - print("b", b) - for x in b['set'].itervalues(): - print(x) - print() - - a['set2'] = set() - a['set2'].add("o1") - a['set2'].add("o1") - a['set2'].add("o2") - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - del b['b'] - try: - print(b['b']) - except KeyError: - print("Yay! deleted key raises error") - - if b.has_key('b'): - print("Boo!") - else: - print("Yay - has_key with delete works!") - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - b.__revertitem__('b') - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - b.__revertitem__('dict') - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py deleted file mode 100644 index 4c7afc9c21..0000000000 --- a/bitbake/lib/bb/__init__.py +++ /dev/null @@ -1,139 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# BitBake Build System Python Library -# -# Copyright (C) 2003 Holger Schurig -# Copyright (C) 2003, 2004 Chris Larson -# -# Based on Gentoo's portage.py. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -__version__ = "1.11.0" - -import sys -if sys.version_info < (2, 6, 0): - raise RuntimeError("Sorry, python 2.6.0 or later is required for this version of bitbake") - -import os -import logging -import traceback - -class NullHandler(logging.Handler): - def emit(self, record): - pass - -Logger = logging.getLoggerClass() -class BBLogger(Logger): - def __init__(self, name): - if name.split(".")[0] == "BitBake": - self.debug = self.bbdebug - Logger.__init__(self, name) - - def bbdebug(self, level, msg, *args, **kwargs): - return self.log(logging.DEBUG - level + 1, msg, *args, **kwargs) - - def plain(self, msg, *args, **kwargs): - return self.log(logging.INFO + 1, msg, *args, **kwargs) - - def verbose(self, msg, *args, **kwargs): - return self.log(logging.INFO - 1, msg, *args, **kwargs) - - def exception(self, msg, *args, **kwargs): - return self.critical("%s\n%s" % (msg, traceback.format_exc()), *args, **kwargs) - -logging.raiseExceptions = False -logging.setLoggerClass(BBLogger) - -logger = logging.getLogger("BitBake") -logger.addHandler(NullHandler()) -logger.setLevel(logging.INFO) - -# This has to be imported after the setLoggerClass, as the import of bb.msg -# can result in construction of the various loggers. -import bb.msg - -if "BBDEBUG" in os.environ: - level = int(os.environ["BBDEBUG"]) - if level: - bb.msg.set_debug_level(level) - -if True or os.environ.get("BBFETCH2"): - from bb import fetch2 as fetch - sys.modules['bb.fetch'] = sys.modules['bb.fetch2'] - -# Messaging convenience functions -def plain(*args): - logger.plain(''.join(args)) - -def debug(lvl, *args): - logger.debug(lvl, ''.join(args)) - -def note(*args): - logger.info(''.join(args)) - -def warn(*args): - logger.warn(''.join(args)) - -def error(*args): - logger.error(''.join(args)) - -def fatal(*args): - logger.critical(''.join(args)) - sys.exit(1) - - -def deprecated(func, name = None, advice = ""): - """This is a decorator which can be used to mark functions - as deprecated. It will result in a warning being emmitted - when the function is used.""" - import warnings - - if advice: - advice = ": %s" % advice - if name is None: - name = func.__name__ - - def newFunc(*args, **kwargs): - warnings.warn("Call to deprecated function %s%s." % (name, - advice), - category = PendingDeprecationWarning, - stacklevel = 2) - return func(*args, **kwargs) - newFunc.__name__ = func.__name__ - newFunc.__doc__ = func.__doc__ - newFunc.__dict__.update(func.__dict__) - return newFunc - -# For compatibility -def deprecate_import(current, modulename, fromlist, renames = None): - """Import objects from one module into another, wrapping them with a DeprecationWarning""" - import sys - - module = __import__(modulename, fromlist = fromlist) - for position, objname in enumerate(fromlist): - obj = getattr(module, objname) - newobj = deprecated(obj, "{0}.{1}".format(current, objname), - "Please use {0}.{1} instead".format(modulename, objname)) - if renames: - newname = renames[position] - else: - newname = objname - - setattr(sys.modules[current], newname, newobj) - -deprecate_import(__name__, "bb.fetch", ("MalformedUrl", "encodeurl", "decodeurl")) -deprecate_import(__name__, "bb.utils", ("mkdirhier", "movefile", "copyfile", "which")) -deprecate_import(__name__, "bb.utils", ["vercmp_string"], ["vercmp"]) diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py deleted file mode 100644 index a7664bd36d..0000000000 --- a/bitbake/lib/bb/build.py +++ /dev/null @@ -1,472 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# BitBake 'Build' implementation -# -# Core code for function execution and task handling in the -# BitBake build tools. -# -# Copyright (C) 2003, 2004 Chris Larson -# -# Based on Gentoo's portage.py. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -#Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import sys -import logging -import bb -import bb.msg -import bb.process -from contextlib import nested -from bb import data, event, mkdirhier, utils - -bblogger = logging.getLogger('BitBake') -logger = logging.getLogger('BitBake.Build') - -NULL = open(os.devnull, 'r+') - - -# When we execute a python function we'd like certain things -# in all namespaces, hence we add them to __builtins__ -# If we do not do this and use the exec globals, they will -# not be available to subfunctions. -__builtins__['bb'] = bb -__builtins__['os'] = os - -class FuncFailed(Exception): - def __init__(self, name = None, logfile = None): - self.logfile = logfile - self.name = name - if name: - self.msg = "Function '%s' failed" % name - else: - self.msg = "Function failed" - - def __str__(self): - if self.logfile and os.path.exists(self.logfile): - msg = ("%s (see %s for further information)" % - (self.msg, self.logfile)) - else: - msg = self.msg - return msg - -class TaskBase(event.Event): - """Base class for task events""" - - def __init__(self, t, d ): - self._task = t - self._package = bb.data.getVar("PF", d, 1) - event.Event.__init__(self) - self._message = "package %s: task %s: %s" % (bb.data.getVar("PF", d, 1), t, bb.event.getName(self)[4:]) - - def getTask(self): - return self._task - - def setTask(self, task): - self._task = task - - task = property(getTask, setTask, None, "task property") - -class TaskStarted(TaskBase): - """Task execution started""" - -class TaskSucceeded(TaskBase): - """Task execution completed""" - -class TaskFailed(TaskBase): - """Task execution failed""" - - def __init__(self, task, logfile, metadata): - self.logfile = logfile - super(TaskFailed, self).__init__(task, metadata) - -class TaskInvalid(TaskBase): - - def __init__(self, task, metadata): - super(TaskInvalid, self).__init__(task, metadata) - self._message = "No such task '%s'" % task - - -class LogTee(object): - def __init__(self, logger, outfile): - self.outfile = outfile - self.logger = logger - self.name = self.outfile.name - - def write(self, string): - self.logger.plain(string) - self.outfile.write(string) - - def __enter__(self): - self.outfile.__enter__() - return self - - def __exit__(self, *excinfo): - self.outfile.__exit__(*excinfo) - - def __repr__(self): - return ''.format(self.name) - - -def exec_func(func, d, dirs = None): - """Execute an BB 'function'""" - - body = data.getVar(func, d) - if not body: - if body is None: - logger.warn("Function %s doesn't exist", func) - return - - flags = data.getVarFlags(func, d) - cleandirs = flags.get('cleandirs') - if cleandirs: - for cdir in data.expand(cleandirs, d).split(): - bb.utils.remove(cdir, True) - - if dirs is None: - dirs = flags.get('dirs') - if dirs: - dirs = data.expand(dirs, d).split() - - if dirs: - for adir in dirs: - bb.utils.mkdirhier(adir) - adir = dirs[-1] - else: - adir = data.getVar('B', d, 1) - if not os.path.exists(adir): - adir = None - - ispython = flags.get('python') - if flags.get('fakeroot') and not flags.get('task'): - bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func) - - lockflag = flags.get('lockfiles') - if lockflag: - lockfiles = [data.expand(f, d) for f in lockflag.split()] - else: - lockfiles = None - - tempdir = data.getVar('T', d, 1) - runfile = os.path.join(tempdir, 'run.{0}.{1}'.format(func, os.getpid())) - - with bb.utils.fileslocked(lockfiles): - if ispython: - exec_func_python(func, d, runfile, cwd=adir) - else: - exec_func_shell(func, d, runfile, cwd=adir) - -_functionfmt = """ -def {function}(d): -{body} - -{function}(d) -""" -logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") -def exec_func_python(func, d, runfile, cwd=None): - """Execute a python BB 'function'""" - - bbfile = d.getVar('FILE', True) - try: - olddir = os.getcwd() - except OSError: - olddir = None - code = _functionfmt.format(function=func, body=d.getVar(func, True)) - bb.utils.mkdirhier(os.path.dirname(runfile)) - with open(runfile, 'w') as script: - script.write(code) - - if cwd: - os.chdir(cwd) - - try: - comp = utils.better_compile(code, func, bbfile) - utils.better_exec(comp, {"d": d}, code, bbfile) - except: - if sys.exc_info()[0] in (bb.parse.SkipPackage, bb.build.FuncFailed): - raise - - raise FuncFailed(func, None) - finally: - if olddir: - os.chdir(olddir) - -def exec_func_shell(function, d, runfile, cwd=None): - """Execute a shell function from the metadata - - Note on directory behavior. The 'dirs' varflag should contain a list - of the directories you need created prior to execution. The last - item in the list is where we will chdir/cd to. - """ - - # Don't let the emitted shell script override PWD - d.delVarFlag('PWD', 'export') - - with open(runfile, 'w') as script: - script.write('#!/bin/sh -e\n') - if logger.isEnabledFor(logging.DEBUG): - script.write("set -x\n") - data.emit_func(function, script, d) - - script.write("%s\n" % function) - os.fchmod(script.fileno(), 0775) - - env = { - 'PATH': d.getVar('PATH', True), - 'LC_ALL': 'C', - } - - cmd = runfile - - if logger.isEnabledFor(logging.DEBUG): - logfile = LogTee(logger, sys.stdout) - else: - logfile = sys.stdout - - try: - bb.process.run(cmd, env=env, cwd=cwd, shell=False, stdin=NULL, - log=logfile) - except bb.process.CmdError: - logfn = d.getVar('BB_LOGFILE', True) - raise FuncFailed(function, logfn) - -def _task_data(fn, task, d): - localdata = data.createCopy(d) - localdata.setVar('BB_FILENAME', fn) - localdata.setVar('BB_CURRENTTASK', task[3:]) - localdata.setVar('OVERRIDES', 'task-%s:%s' % - (task[3:], d.getVar('OVERRIDES', False))) - localdata.finalize() - data.expandKeys(localdata) - return localdata - -def _exec_task(fn, task, d, quieterr): - """Execute a BB 'task' - - Execution of a task involves a bit more setup than executing a function, - running it with its own local metadata, and with some useful variables set. - """ - if not data.getVarFlag(task, 'task', d): - event.fire(TaskInvalid(task, d), d) - logger.error("No such task: %s" % task) - return 1 - - logger.debug(1, "Executing task %s", task) - - localdata = _task_data(fn, task, d) - tempdir = localdata.getVar('T', True) - if not tempdir: - bb.fatal("T variable not set, unable to build") - - bb.utils.mkdirhier(tempdir) - loglink = os.path.join(tempdir, 'log.{0}'.format(task)) - logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid())) - if loglink: - bb.utils.remove(loglink) - - try: - os.symlink(logfn, loglink) - except OSError: - pass - - prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) - postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) - - # Handle logfiles - si = file('/dev/null', 'r') - try: - logfile = file(logfn, 'w') - except OSError: - logger.exception("Opening log file '%s'", logfn) - pass - - # Dup the existing fds so we dont lose them - osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] - oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] - ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] - - # Replace those fds with our own - os.dup2(si.fileno(), osi[1]) - os.dup2(logfile.fileno(), oso[1]) - os.dup2(logfile.fileno(), ose[1]) - - # Ensure python logging goes to the logfile - handler = logging.StreamHandler(logfile) - handler.setFormatter(logformatter) - bblogger.addHandler(handler) - - localdata.setVar('BB_LOGFILE', logfn) - - event.fire(TaskStarted(task, localdata), localdata) - try: - for func in (prefuncs or '').split(): - exec_func(func, localdata) - exec_func(task, localdata) - for func in (postfuncs or '').split(): - exec_func(func, localdata) - except FuncFailed as exc: - if not quieterr: - logger.error(str(exc)) - event.fire(TaskFailed(task, logfn, localdata), localdata) - return 1 - finally: - sys.stdout.flush() - sys.stderr.flush() - - bblogger.removeHandler(handler) - - # Restore the backup fds - os.dup2(osi[0], osi[1]) - os.dup2(oso[0], oso[1]) - os.dup2(ose[0], ose[1]) - - # Close the backup fds - os.close(osi[0]) - os.close(oso[0]) - os.close(ose[0]) - si.close() - - logfile.close() - if os.path.exists(logfn) and os.path.getsize(logfn) == 0: - logger.debug(2, "Zero size logfn %s, removing", logfn) - bb.utils.remove(logfn) - bb.utils.remove(loglink) - event.fire(TaskSucceeded(task, localdata), localdata) - - if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'): - make_stamp(task, localdata) - - return 0 - -def exec_task(fn, task, d): - try: - quieterr = False - if d.getVarFlag(task, "quieterrors") is not None: - quieterr = True - - return _exec_task(fn, task, d, quieterr) - except Exception: - from traceback import format_exc - if not quieterr: - logger.error("Build of %s failed" % (task)) - logger.error(format_exc()) - failedevent = TaskFailed(task, None, d) - event.fire(failedevent, d) - return 1 - -def stamp_internal(taskname, d, file_name): - """ - Internal stamp helper function - Makes sure the stamp directory exists - Returns the stamp path+filename - - In the bitbake core, d can be a CacheData and file_name will be set. - When called in task context, d will be a data store, file_name will not be set - """ - taskflagname = taskname - if taskname.endswith("_setscene") and taskname != "do_setscene": - taskflagname = taskname.replace("_setscene", "") - - if file_name: - stamp = d.stamp[file_name] - extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" - else: - stamp = d.getVar('STAMP', True) - file_name = d.getVar('BB_FILENAME', True) - extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or "" - - if not stamp: - return - - stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo) - - bb.utils.mkdirhier(os.path.dirname(stamp)) - - return stamp - -def make_stamp(task, d, file_name = None): - """ - Creates/updates a stamp for a given task - (d can be a data dict or dataCache) - """ - stamp = stamp_internal(task, d, file_name) - # Remove the file and recreate to force timestamp - # change on broken NFS filesystems - if stamp: - bb.utils.remove(stamp) - f = open(stamp, "w") - f.close() - -def del_stamp(task, d, file_name = None): - """ - Removes a stamp for a given task - (d can be a data dict or dataCache) - """ - stamp = stamp_internal(task, d, file_name) - bb.utils.remove(stamp) - -def stampfile(taskname, d, file_name = None): - """ - Return the stamp for a given task - (d can be a data dict or dataCache) - """ - return stamp_internal(taskname, d, file_name) - -def add_tasks(tasklist, d): - task_deps = data.getVar('_task_deps', d) - if not task_deps: - task_deps = {} - if not 'tasks' in task_deps: - task_deps['tasks'] = [] - if not 'parents' in task_deps: - task_deps['parents'] = {} - - for task in tasklist: - task = data.expand(task, d) - data.setVarFlag(task, 'task', 1, d) - - if not task in task_deps['tasks']: - task_deps['tasks'].append(task) - - flags = data.getVarFlags(task, d) - def getTask(name): - if not name in task_deps: - task_deps[name] = {} - if name in flags: - deptask = data.expand(flags[name], d) - task_deps[name][task] = deptask - getTask('depends') - getTask('deptask') - getTask('rdeptask') - getTask('recrdeptask') - getTask('nostamp') - getTask('fakeroot') - getTask('noexec') - task_deps['parents'][task] = [] - for dep in flags['deps']: - dep = data.expand(dep, d) - task_deps['parents'][task].append(dep) - - # don't assume holding a reference - data.setVar('_task_deps', task_deps, d) - -def remove_task(task, kill, d): - """Remove an BB 'task'. - - If kill is 1, also remove tasks that depend on this task.""" - - data.delVarFlag(task, 'task', d) diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py deleted file mode 100644 index 7ea04ac1a5..0000000000 --- a/bitbake/lib/bb/cache.py +++ /dev/null @@ -1,632 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# BitBake 'Event' implementation -# -# Caching of bitbake variables before task execution - -# Copyright (C) 2006 Richard Purdie - -# but small sections based on code from bin/bitbake: -# Copyright (C) 2003, 2004 Chris Larson -# Copyright (C) 2003, 2004 Phil Blundell -# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer -# Copyright (C) 2005 Holger Hans Peter Freyther -# Copyright (C) 2005 ROAD GmbH -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import os -import logging -from collections import defaultdict, namedtuple -import bb.data -import bb.utils - -logger = logging.getLogger("BitBake.Cache") - -try: - import cPickle as pickle -except ImportError: - import pickle - logger.info("Importing cPickle failed. " - "Falling back to a very slow implementation.") - -__cache_version__ = "136" - -recipe_fields = ( - 'pn', - 'pv', - 'pr', - 'pe', - 'defaultpref', - 'depends', - 'provides', - 'task_deps', - 'stamp', - 'stamp_extrainfo', - 'broken', - 'not_world', - 'skipped', - 'timestamp', - 'packages', - 'packages_dynamic', - 'rdepends', - 'rdepends_pkg', - 'rprovides', - 'rprovides_pkg', - 'rrecommends', - 'rrecommends_pkg', - 'nocache', - 'variants', - 'file_depends', - 'tasks', - 'basetaskhashes', - 'hashfilename', -) - - -class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)): - __slots__ = () - - @classmethod - def listvar(cls, var, metadata): - return cls.getvar(var, metadata).split() - - @classmethod - def intvar(cls, var, metadata): - return int(cls.getvar(var, metadata) or 0) - - @classmethod - def depvar(cls, var, metadata): - return bb.utils.explode_deps(cls.getvar(var, metadata)) - - @classmethod - def pkgvar(cls, var, packages, metadata): - return dict((pkg, cls.depvar("%s_%s" % (var, pkg), metadata)) - for pkg in packages) - - @classmethod - def taskvar(cls, var, tasks, metadata): - return dict((task, cls.getvar("%s_task-%s" % (var, task), metadata)) - for task in tasks) - - @classmethod - def flaglist(cls, flag, varlist, metadata): - return dict((var, metadata.getVarFlag(var, flag, True)) - for var in varlist) - - @classmethod - def getvar(cls, var, metadata): - return metadata.getVar(var, True) or '' - - @classmethod - def make_optional(cls, default=None, **kwargs): - """Construct the namedtuple from the specified keyword arguments, - with every value considered optional, using the default value if - it was not specified.""" - for field in cls._fields: - kwargs[field] = kwargs.get(field, default) - return cls(**kwargs) - - @classmethod - def from_metadata(cls, filename, metadata): - if cls.getvar('__SKIPPED', metadata): - return cls.make_optional(skipped=True) - - tasks = metadata.getVar('__BBTASKS', False) - - pn = cls.getvar('PN', metadata) - packages = cls.listvar('PACKAGES', metadata) - if not pn in packages: - packages.append(pn) - - return RecipeInfo( - tasks = tasks, - basetaskhashes = cls.taskvar('BB_BASEHASH', tasks, metadata), - hashfilename = cls.getvar('BB_HASHFILENAME', metadata), - - file_depends = metadata.getVar('__depends', False), - task_deps = metadata.getVar('_task_deps', False) or - {'tasks': [], 'parents': {}}, - variants = cls.listvar('__VARIANTS', metadata) + [''], - - skipped = False, - timestamp = bb.parse.cached_mtime(filename), - packages = cls.listvar('PACKAGES', metadata), - pn = pn, - pe = cls.getvar('PE', metadata), - pv = cls.getvar('PV', metadata), - pr = cls.getvar('PR', metadata), - nocache = cls.getvar('__BB_DONT_CACHE', metadata), - defaultpref = cls.intvar('DEFAULT_PREFERENCE', metadata), - broken = cls.getvar('BROKEN', metadata), - not_world = cls.getvar('EXCLUDE_FROM_WORLD', metadata), - stamp = cls.getvar('STAMP', metadata), - stamp_extrainfo = cls.flaglist('stamp-extra-info', tasks, metadata), - packages_dynamic = cls.listvar('PACKAGES_DYNAMIC', metadata), - depends = cls.depvar('DEPENDS', metadata), - provides = cls.depvar('PROVIDES', metadata), - rdepends = cls.depvar('RDEPENDS', metadata), - rprovides = cls.depvar('RPROVIDES', metadata), - rrecommends = cls.depvar('RRECOMMENDS', metadata), - rprovides_pkg = cls.pkgvar('RPROVIDES', packages, metadata), - rdepends_pkg = cls.pkgvar('RDEPENDS', packages, metadata), - rrecommends_pkg = cls.pkgvar('RRECOMMENDS', packages, metadata), - ) - - -class Cache(object): - """ - BitBake Cache implementation - """ - - def __init__(self, data): - self.cachedir = bb.data.getVar("CACHE", data, True) - self.clean = set() - self.checked = set() - self.depends_cache = {} - self.data = None - self.data_fn = None - self.cacheclean = True - - if self.cachedir in [None, '']: - self.has_cache = False - logger.info("Not using a cache. " - "Set CACHE = to enable.") - return - - self.has_cache = True - self.cachefile = os.path.join(self.cachedir, "bb_cache.dat") - - logger.debug(1, "Using cache in '%s'", self.cachedir) - bb.utils.mkdirhier(self.cachedir) - - # If any of configuration.data's dependencies are newer than the - # cache there isn't even any point in loading it... - newest_mtime = 0 - deps = bb.data.getVar("__base_depends", data) - - old_mtimes = [old_mtime for _, old_mtime in deps] - old_mtimes.append(newest_mtime) - newest_mtime = max(old_mtimes) - - if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: - self.load_cachefile() - elif os.path.isfile(self.cachefile): - logger.info("Out of date cache found, rebuilding...") - - def load_cachefile(self): - with open(self.cachefile, "rb") as cachefile: - pickled = pickle.Unpickler(cachefile) - try: - cache_ver = pickled.load() - bitbake_ver = pickled.load() - except Exception: - logger.info('Invalid cache, rebuilding...') - return - - if cache_ver != __cache_version__: - logger.info('Cache version mismatch, rebuilding...') - return - elif bitbake_ver != bb.__version__: - logger.info('Bitbake version mismatch, rebuilding...') - return - - cachesize = os.fstat(cachefile.fileno()).st_size - bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) - - previous_percent = 0 - while cachefile: - try: - key = pickled.load() - value = pickled.load() - except Exception: - break - - self.depends_cache[key] = value - - # only fire events on even percentage boundaries - current_progress = cachefile.tell() - current_percent = 100 * current_progress / cachesize - if current_percent > previous_percent: - previous_percent = current_percent - bb.event.fire(bb.event.CacheLoadProgress(current_progress), - self.data) - - bb.event.fire(bb.event.CacheLoadCompleted(cachesize, - len(self.depends_cache)), - self.data) - - @staticmethod - def virtualfn2realfn(virtualfn): - """ - Convert a virtual file name to a real one + the associated subclass keyword - """ - - fn = virtualfn - cls = "" - if virtualfn.startswith('virtual:'): - cls = virtualfn.split(':', 2)[1] - fn = virtualfn.replace('virtual:' + cls + ':', '') - return (fn, cls) - - @staticmethod - def realfn2virtual(realfn, cls): - """ - Convert a real filename + the associated subclass keyword to a virtual filename - """ - if cls == "": - return realfn - return "virtual:" + cls + ":" + realfn - - @classmethod - def loadDataFull(cls, virtualfn, appends, cfgData): - """ - Return a complete set of data for fn. - To do this, we need to parse the file. - """ - - (fn, virtual) = cls.virtualfn2realfn(virtualfn) - - logger.debug(1, "Parsing %s (full)", fn) - - bb_data = cls.load_bbfile(fn, appends, cfgData) - return bb_data[virtual] - - @classmethod - def parse(cls, filename, appends, configdata): - """Parse the specified filename, returning the recipe information""" - infos = [] - datastores = cls.load_bbfile(filename, appends, configdata) - depends = set() - for variant, data in sorted(datastores.iteritems(), - key=lambda i: i[0], - reverse=True): - virtualfn = cls.realfn2virtual(filename, variant) - depends |= (data.getVar("__depends", False) or set()) - if depends and not variant: - data.setVar("__depends", depends) - info = RecipeInfo.from_metadata(filename, data) - infos.append((virtualfn, info)) - return infos - - def load(self, filename, appends, configdata): - """Obtain the recipe information for the specified filename, - using cached values if available, otherwise parsing. - - Note that if it does parse to obtain the info, it will not - automatically add the information to the cache or to your - CacheData. Use the add or add_info method to do so after - running this, or use loadData instead.""" - cached = self.cacheValid(filename) - if cached: - infos = [] - info = self.depends_cache[filename] - for variant in info.variants: - virtualfn = self.realfn2virtual(filename, variant) - infos.append((virtualfn, self.depends_cache[virtualfn])) - else: - logger.debug(1, "Parsing %s", filename) - return self.parse(filename, appends, configdata) - - return cached, infos - - def loadData(self, fn, appends, cfgData, cacheData): - """Load the recipe info for the specified filename, - parsing and adding to the cache if necessary, and adding - the recipe information to the supplied CacheData instance.""" - skipped, virtuals = 0, 0 - - cached, infos = self.load(fn, appends, cfgData) - for virtualfn, info in infos: - if info.skipped: - logger.debug(1, "Skipping %s", virtualfn) - skipped += 1 - else: - self.add_info(virtualfn, info, cacheData, not cached) - virtuals += 1 - - return cached, skipped, virtuals - - def cacheValid(self, fn): - """ - Is the cache valid for fn? - Fast version, no timestamps checked. - """ - if fn not in self.checked: - self.cacheValidUpdate(fn) - - # Is cache enabled? - if not self.has_cache: - return False - if fn in self.clean: - return True - return False - - def cacheValidUpdate(self, fn): - """ - Is the cache valid for fn? - Make thorough (slower) checks including timestamps. - """ - # Is cache enabled? - if not self.has_cache: - return False - - self.checked.add(fn) - - # File isn't in depends_cache - if not fn in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", fn) - return False - - mtime = bb.parse.cached_mtime_noerror(fn) - - # Check file still exists - if mtime == 0: - logger.debug(2, "Cache: %s no longer exists", fn) - self.remove(fn) - return False - - info = self.depends_cache[fn] - # Check the file's timestamp - if mtime != info.timestamp: - logger.debug(2, "Cache: %s changed", fn) - self.remove(fn) - return False - - # Check dependencies are still valid - depends = info.file_depends - if depends: - for f, old_mtime in depends: - fmtime = bb.parse.cached_mtime_noerror(f) - # Check if file still exists - if old_mtime != 0 and fmtime == 0: - logger.debug(2, "Cache: %s's dependency %s was removed", - fn, f) - self.remove(fn) - return False - - if (fmtime != old_mtime): - logger.debug(2, "Cache: %s's dependency %s changed", - fn, f) - self.remove(fn) - return False - - invalid = False - for cls in info.variants: - virtualfn = self.realfn2virtual(fn, cls) - self.clean.add(virtualfn) - if virtualfn not in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", virtualfn) - invalid = True - - # If any one of the variants is not present, mark as invalid for all - if invalid: - for cls in info.variants: - virtualfn = self.realfn2virtual(fn, cls) - if virtualfn in self.clean: - logger.debug(2, "Cache: Removing %s from cache", virtualfn) - self.clean.remove(virtualfn) - if fn in self.clean: - logger.debug(2, "Cache: Marking %s as not clean", fn) - self.clean.remove(fn) - return False - - self.clean.add(fn) - return True - - def remove(self, fn): - """ - Remove a fn from the cache - Called from the parser in error cases - """ - if fn in self.depends_cache: - logger.debug(1, "Removing %s from cache", fn) - del self.depends_cache[fn] - if fn in self.clean: - logger.debug(1, "Marking %s as unclean", fn) - self.clean.remove(fn) - - def sync(self): - """ - Save the cache - Called from the parser when complete (or exiting) - """ - - if not self.has_cache: - return - - if self.cacheclean: - logger.debug(2, "Cache is clean, not saving.") - return - - with open(self.cachefile, "wb") as cachefile: - pickler = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL) - pickler.dump(__cache_version__) - pickler.dump(bb.__version__) - for key, value in self.depends_cache.iteritems(): - pickler.dump(key) - pickler.dump(value) - - del self.depends_cache - - @staticmethod - def mtime(cachefile): - return bb.parse.cached_mtime_noerror(cachefile) - - def add_info(self, filename, info, cacheData, parsed=None): - cacheData.add_from_recipeinfo(filename, info) - if not self.has_cache: - return - - if 'SRCREVINACTION' not in info.pv and not info.nocache: - if parsed: - self.cacheclean = False - self.depends_cache[filename] = info - - def add(self, file_name, data, cacheData, parsed=None): - """ - Save data we need into the cache - """ - - realfn = self.virtualfn2realfn(file_name)[0] - info = RecipeInfo.from_metadata(realfn, data) - self.add_info(file_name, info, cacheData, parsed) - - @staticmethod - def load_bbfile(bbfile, appends, config): - """ - Load and parse one .bb build file - Return the data and whether parsing resulted in the file being skipped - """ - chdir_back = False - - from bb import data, parse - - # expand tmpdir to include this topdir - data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config) - bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) - oldpath = os.path.abspath(os.getcwd()) - parse.cached_mtime_noerror(bbfile_loc) - bb_data = data.init_db(config) - # The ConfHandler first looks if there is a TOPDIR and if not - # then it would call getcwd(). - # Previously, we chdir()ed to bbfile_loc, called the handler - # and finally chdir()ed back, a couple of thousand times. We now - # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet. - if not data.getVar('TOPDIR', bb_data): - chdir_back = True - data.setVar('TOPDIR', bbfile_loc, bb_data) - try: - if appends: - data.setVar('__BBAPPEND', " ".join(appends), bb_data) - bb_data = parse.handle(bbfile, bb_data) - if chdir_back: - os.chdir(oldpath) - return bb_data - except: - if chdir_back: - os.chdir(oldpath) - raise - - -def init(cooker): - """ - The Objective: Cache the minimum amount of data possible yet get to the - stage of building packages (i.e. tryBuild) without reparsing any .bb files. - - To do this, we intercept getVar calls and only cache the variables we see - being accessed. We rely on the cache getVar calls being made for all - variables bitbake might need to use to reach this stage. For each cached - file we need to track: - - * Its mtime - * The mtimes of all its dependencies - * Whether it caused a parse.SkipPackage exception - - Files causing parsing errors are evicted from the cache. - - """ - return Cache(cooker.configuration.data) - - -class CacheData(object): - """ - The data structures we compile from the cached data - """ - - def __init__(self): - # Direct cache variables - self.providers = defaultdict(list) - self.rproviders = defaultdict(list) - self.packages = defaultdict(list) - self.packages_dynamic = defaultdict(list) - self.possible_world = [] - self.pkg_pn = defaultdict(list) - self.pkg_fn = {} - self.pkg_pepvpr = {} - self.pkg_dp = {} - self.pn_provides = defaultdict(list) - self.fn_provides = {} - self.all_depends = [] - self.deps = defaultdict(list) - self.rundeps = defaultdict(lambda: defaultdict(list)) - self.runrecs = defaultdict(lambda: defaultdict(list)) - self.task_queues = {} - self.task_deps = {} - self.stamp = {} - self.stamp_extrainfo = {} - self.preferred = {} - self.tasks = {} - self.basetaskhash = {} - self.hashfn = {} - - # Indirect Cache variables (set elsewhere) - self.ignored_dependencies = [] - self.world_target = set() - self.bbfile_priority = {} - self.bbfile_config_priorities = [] - - def add_from_recipeinfo(self, fn, info): - self.task_deps[fn] = info.task_deps - self.pkg_fn[fn] = info.pn - self.pkg_pn[info.pn].append(fn) - self.pkg_pepvpr[fn] = (info.pe, info.pv, info.pr) - self.pkg_dp[fn] = info.defaultpref - self.stamp[fn] = info.stamp - self.stamp_extrainfo[fn] = info.stamp_extrainfo - - provides = [info.pn] - for provide in info.provides: - if provide not in provides: - provides.append(provide) - self.fn_provides[fn] = provides - - for provide in provides: - self.providers[provide].append(fn) - if provide not in self.pn_provides[info.pn]: - self.pn_provides[info.pn].append(provide) - - for dep in info.depends: - if dep not in self.deps[fn]: - self.deps[fn].append(dep) - if dep not in self.all_depends: - self.all_depends.append(dep) - - rprovides = info.rprovides - for package in info.packages: - self.packages[package].append(fn) - rprovides += info.rprovides_pkg[package] - - for rprovide in rprovides: - self.rproviders[rprovide].append(fn) - - for package in info.packages_dynamic: - self.packages_dynamic[package].append(fn) - - # Build hash of runtime depends and rececommends - for package in info.packages + [info.pn]: - self.rundeps[fn][package] = list(info.rdepends) + info.rdepends_pkg[package] - self.runrecs[fn][package] = list(info.rrecommends) + info.rrecommends_pkg[package] - - # Collect files we may need for possible world-dep - # calculations - if not info.broken and not info.not_world: - self.possible_world.append(fn) - - self.hashfn[fn] = info.hashfilename - for task, taskhash in info.basetaskhashes.iteritems(): - identifier = '%s.%s' % (fn, task) - self.basetaskhash[identifier] = taskhash diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py deleted file mode 100644 index bfffcacc33..0000000000 --- a/bitbake/lib/bb/codeparser.py +++ /dev/null @@ -1,336 +0,0 @@ -import ast -import codegen -import logging -import os.path -import bb.utils, bb.data -from itertools import chain -from pysh import pyshyacc, pyshlex - - -logger = logging.getLogger('BitBake.CodeParser') -PARSERCACHE_VERSION = 2 - -try: - import cPickle as pickle -except ImportError: - import pickle - logger.info('Importing cPickle failed. Falling back to a very slow implementation.') - - -def check_indent(codestr): - """If the code is indented, add a top level piece of code to 'remove' the indentation""" - - i = 0 - while codestr[i] in ["\n", " ", " "]: - i = i + 1 - - if i == 0: - return codestr - - if codestr[i-1] is " " or codestr[i-1] is " ": - return "if 1:\n" + codestr - - return codestr - -pythonparsecache = {} -shellparsecache = {} - -def parser_cachefile(d): - cachedir = (bb.data.getVar("PERSISTENT_DIR", d, True) or - bb.data.getVar("CACHE", d, True)) - if cachedir in [None, '']: - return None - bb.utils.mkdirhier(cachedir) - cachefile = os.path.join(cachedir, "bb_codeparser.dat") - logger.debug(1, "Using cache in '%s' for codeparser cache", cachefile) - return cachefile - -def parser_cache_init(d): - global pythonparsecache - global shellparsecache - - cachefile = parser_cachefile(d) - if not cachefile: - return - - try: - p = pickle.Unpickler(file(cachefile, "rb")) - data, version = p.load() - except: - return - - if version != PARSERCACHE_VERSION: - return - - pythonparsecache = data[0] - shellparsecache = data[1] - -def parser_cache_save(d): - cachefile = parser_cachefile(d) - if not cachefile: - return - - p = pickle.Pickler(file(cachefile, "wb"), -1) - p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION]) - -class PythonParser(): - class ValueVisitor(): - """Visitor to traverse a python abstract syntax tree and obtain - the variables referenced via bitbake metadata APIs, and the external - functions called. - """ - - getvars = ("d.getVar", "bb.data.getVar", "data.getVar") - expands = ("d.expand", "bb.data.expand", "data.expand") - execs = ("bb.build.exec_func", "bb.build.exec_task") - - @classmethod - def _compare_name(cls, strparts, node): - """Given a sequence of strings representing a python name, - where the last component is the actual Name and the prior - elements are Attribute nodes, determine if the supplied node - matches. - """ - - if not strparts: - return True - - current, rest = strparts[0], strparts[1:] - if isinstance(node, ast.Attribute): - if current == node.attr: - return cls._compare_name(rest, node.value) - elif isinstance(node, ast.Name): - if current == node.id: - return True - return False - - @classmethod - def compare_name(cls, value, node): - """Convenience function for the _compare_node method, which - can accept a string (which is split by '.' for you), or an - iterable of strings, in which case it checks to see if any of - them match, similar to isinstance. - """ - - if isinstance(value, basestring): - return cls._compare_name(tuple(reversed(value.split("."))), - node) - else: - return any(cls.compare_name(item, node) for item in value) - - def __init__(self, value): - self.var_references = set() - self.var_execs = set() - self.direct_func_calls = set() - self.var_expands = set() - self.value = value - - @classmethod - def warn(cls, func, arg): - """Warn about calls of bitbake APIs which pass a non-literal - argument for the variable name, as we're not able to track such - a reference. - """ - - try: - funcstr = codegen.to_source(func) - argstr = codegen.to_source(arg) - except TypeError: - logger.debug(2, 'Failed to convert function and argument to source form') - else: - logger.debug(1, "Warning: in call to '%s', argument '%s' is " - "not a literal", funcstr, argstr) - - def visit_Call(self, node): - if self.compare_name(self.getvars, node.func): - if isinstance(node.args[0], ast.Str): - self.var_references.add(node.args[0].s) - else: - self.warn(node.func, node.args[0]) - elif self.compare_name(self.expands, node.func): - if isinstance(node.args[0], ast.Str): - self.warn(node.func, node.args[0]) - self.var_expands.update(node.args[0].s) - elif isinstance(node.args[0], ast.Call) and \ - self.compare_name(self.get