From c30eddb243e7e65f67f656e62848a033cf6f2e5c Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Tue, 30 Sep 2008 15:08:33 +0000 Subject: Add bitbake-dev to allow ease of testing and development of bitbake trunk git-svn-id: https://svn.o-hand.com/repos/poky/trunk@5337 311d38ba-8fff-0310-9ca6-ca027cbcb966 --- bitbake-dev/lib/bb/COW.py | 320 ++++++ bitbake-dev/lib/bb/__init__.py | 1133 +++++++++++++++++++++ bitbake-dev/lib/bb/build.py | 377 +++++++ bitbake-dev/lib/bb/cache.py | 465 +++++++++ bitbake-dev/lib/bb/command.py | 211 ++++ bitbake-dev/lib/bb/cooker.py | 941 ++++++++++++++++++ bitbake-dev/lib/bb/daemonize.py | 189 ++++ bitbake-dev/lib/bb/data.py | 570 +++++++++++ bitbake-dev/lib/bb/data_smart.py | 292 ++++++ bitbake-dev/lib/bb/event.py | 302 ++++++ bitbake-dev/lib/bb/fetch/__init__.py | 556 +++++++++++ bitbake-dev/lib/bb/fetch/bzr.py | 154 +++ bitbake-dev/lib/bb/fetch/cvs.py | 178 ++++ bitbake-dev/lib/bb/fetch/git.py | 142 +++ bitbake-dev/lib/bb/fetch/hg.py | 141 +++ bitbake-dev/lib/bb/fetch/local.py | 72 ++ bitbake-dev/lib/bb/fetch/perforce.py | 213 ++++ bitbake-dev/lib/bb/fetch/ssh.py | 120 +++ bitbake-dev/lib/bb/fetch/svk.py | 109 ++ bitbake-dev/lib/bb/fetch/svn.py | 204 ++++ bitbake-dev/lib/bb/fetch/wget.py | 105 ++ bitbake-dev/lib/bb/manifest.py | 144 +++ bitbake-dev/lib/bb/methodpool.py | 84 ++ bitbake-dev/lib/bb/msg.py | 125 +++ bitbake-dev/lib/bb/parse/__init__.py | 80 ++ bitbake-dev/lib/bb/parse/parse_py/BBHandler.py | 416 ++++++++ bitbake-dev/lib/bb/parse/parse_py/ConfHandler.py | 228 +++++ bitbake-dev/lib/bb/parse/parse_py/__init__.py | 33 + bitbake-dev/lib/bb/persist_data.py | 110 ++ bitbake-dev/lib/bb/providers.py | 303 ++++++ bitbake-dev/lib/bb/runqueue.py | 1157 ++++++++++++++++++++++ bitbake-dev/lib/bb/shell.py | 827 ++++++++++++++++ bitbake-dev/lib/bb/taskdata.py | 594 +++++++++++ bitbake-dev/lib/bb/ui/__init__.py | 18 + bitbake-dev/lib/bb/ui/depexplorer.py | 271 +++++ bitbake-dev/lib/bb/ui/knotty.py | 157 +++ bitbake-dev/lib/bb/ui/ncurses.py | 333 +++++++ bitbake-dev/lib/bb/ui/uievent.py | 127 +++ bitbake-dev/lib/bb/ui/uihelper.py | 49 + bitbake-dev/lib/bb/utils.py | 270 +++++ bitbake-dev/lib/bb/xmlrpcserver.py | 157 +++ 41 files changed, 12277 insertions(+) create mode 100644 bitbake-dev/lib/bb/COW.py create mode 100644 bitbake-dev/lib/bb/__init__.py create mode 100644 bitbake-dev/lib/bb/build.py create mode 100644 bitbake-dev/lib/bb/cache.py create mode 100644 bitbake-dev/lib/bb/command.py create mode 100644 bitbake-dev/lib/bb/cooker.py create mode 100644 bitbake-dev/lib/bb/daemonize.py create mode 100644 bitbake-dev/lib/bb/data.py create mode 100644 bitbake-dev/lib/bb/data_smart.py create mode 100644 bitbake-dev/lib/bb/event.py create mode 100644 bitbake-dev/lib/bb/fetch/__init__.py create mode 100644 bitbake-dev/lib/bb/fetch/bzr.py create mode 100644 bitbake-dev/lib/bb/fetch/cvs.py create mode 100644 bitbake-dev/lib/bb/fetch/git.py create mode 100644 bitbake-dev/lib/bb/fetch/hg.py create mode 100644 bitbake-dev/lib/bb/fetch/local.py create mode 100644 bitbake-dev/lib/bb/fetch/perforce.py create mode 100644 bitbake-dev/lib/bb/fetch/ssh.py create mode 100644 bitbake-dev/lib/bb/fetch/svk.py create mode 100644 bitbake-dev/lib/bb/fetch/svn.py create mode 100644 bitbake-dev/lib/bb/fetch/wget.py create mode 100644 bitbake-dev/lib/bb/manifest.py create mode 100644 bitbake-dev/lib/bb/methodpool.py create mode 100644 bitbake-dev/lib/bb/msg.py create mode 100644 bitbake-dev/lib/bb/parse/__init__.py create mode 100644 bitbake-dev/lib/bb/parse/parse_py/BBHandler.py create mode 100644 bitbake-dev/lib/bb/parse/parse_py/ConfHandler.py create mode 100644 bitbake-dev/lib/bb/parse/parse_py/__init__.py create mode 100644 bitbake-dev/lib/bb/persist_data.py create mode 100644 bitbake-dev/lib/bb/providers.py create mode 100644 bitbake-dev/lib/bb/runqueue.py create mode 100644 bitbake-dev/lib/bb/shell.py create mode 100644 bitbake-dev/lib/bb/taskdata.py create mode 100644 bitbake-dev/lib/bb/ui/__init__.py create mode 100644 bitbake-dev/lib/bb/ui/depexplorer.py create mode 100644 bitbake-dev/lib/bb/ui/knotty.py create mode 100644 bitbake-dev/lib/bb/ui/ncurses.py create mode 100644 bitbake-dev/lib/bb/ui/uievent.py create mode 100644 bitbake-dev/lib/bb/ui/uihelper.py create mode 100644 bitbake-dev/lib/bb/utils.py create mode 100644 bitbake-dev/lib/bb/xmlrpcserver.py (limited to 'bitbake-dev/lib/bb') diff --git a/bitbake-dev/lib/bb/COW.py b/bitbake-dev/lib/bb/COW.py new file mode 100644 index 0000000000..e5063d60a8 --- /dev/null +++ b/bitbake-dev/lib/bb/COW.py @@ -0,0 +1,320 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# This is a copy on write dictionary and set which abuses classes to try and be nice and fast. +# +# Copyright (C) 2006 Tim Amsell +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +#Please Note: +# Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW. +# Assign a file to __warn__ to get warnings about slow operations. +# + +from inspect import getmro + +import copy +import types, sets +types.ImmutableTypes = tuple([ \ + types.BooleanType, \ + types.ComplexType, \ + types.FloatType, \ + types.IntType, \ + types.LongType, \ + types.NoneType, \ + types.TupleType, \ + sets.ImmutableSet] + \ + list(types.StringTypes)) + +MUTABLE = "__mutable__" + +class COWMeta(type): + pass + +class COWDictMeta(COWMeta): + __warn__ = False + __hasmutable__ = False + __marker__ = tuple() + + def __str__(cls): + # FIXME: I have magic numbers! + return "" % (cls.__count__, len(cls.__dict__) - 3) + __repr__ = __str__ + + def cow(cls): + class C(cls): + __count__ = cls.__count__ + 1 + return C + copy = cow + __call__ = cow + + def __setitem__(cls, key, value): + if not isinstance(value, types.ImmutableTypes): + if not isinstance(value, COWMeta): + cls.__hasmutable__ = True + key += MUTABLE + setattr(cls, key, value) + + def __getmutable__(cls, key, readonly=False): + nkey = key + MUTABLE + try: + return cls.__dict__[nkey] + except KeyError: + pass + + value = getattr(cls, nkey) + if readonly: + return value + + if not cls.__warn__ is False and not isinstance(value, COWMeta): + print >> cls.__warn__, "Warning: Doing a copy because %s is a mutable type." % key + try: + value = value.copy() + except AttributeError, e: + value = copy.copy(value) + setattr(cls, nkey, value) + return value + + __getmarker__ = [] + def __getreadonly__(cls, key, default=__getmarker__): + """\ + Get a value (even if mutable) which you promise not to change. + """ + return cls.__getitem__(key, default, True) + + def __getitem__(cls, key, default=__getmarker__, readonly=False): + try: + try: + value = getattr(cls, key) + except AttributeError: + value = cls.__getmutable__(key, readonly) + + # This is for values which have been deleted + if value is cls.__marker__: + raise AttributeError("key %s does not exist." % key) + + return value + except AttributeError, e: + if not default is cls.__getmarker__: + return default + + raise KeyError(str(e)) + + def __delitem__(cls, key): + cls.__setitem__(key, cls.__marker__) + + def __revertitem__(cls, key): + if not cls.__dict__.has_key(key): + key += MUTABLE + delattr(cls, key) + + def has_key(cls, key): + value = cls.__getreadonly__(key, cls.__marker__) + if value is cls.__marker__: + return False + return True + + def iter(cls, type, readonly=False): + for key in dir(cls): + if key.startswith("__"): + continue + + if key.endswith(MUTABLE): + key = key[:-len(MUTABLE)] + + if type == "keys": + yield key + + try: + if readonly: + value = cls.__getreadonly__(key) + else: + value = cls[key] + except KeyError: + continue + + if type == "values": + yield value + if type == "items": + yield (key, value) + raise StopIteration() + + def iterkeys(cls): + return cls.iter("keys") + def itervalues(cls, readonly=False): + if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: + print >> cls.__warn__, "Warning: If you arn't going to change any of the values call with True." + return cls.iter("values", readonly) + def iteritems(cls, readonly=False): + if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: + print >> cls.__warn__, "Warning: If you arn't going to change any of the values call with True." + return cls.iter("items", readonly) + +class COWSetMeta(COWDictMeta): + def __str__(cls): + # FIXME: I have magic numbers! + return "" % (cls.__count__, len(cls.__dict__) -3) + __repr__ = __str__ + + def cow(cls): + class C(cls): + __count__ = cls.__count__ + 1 + return C + + def add(cls, value): + COWDictMeta.__setitem__(cls, repr(hash(value)), value) + + def remove(cls, value): + COWDictMeta.__delitem__(cls, repr(hash(value))) + + def __in__(cls, value): + return COWDictMeta.has_key(repr(hash(value))) + + def iterkeys(cls): + raise TypeError("sets don't have keys") + + def iteritems(cls): + raise TypeError("sets don't have 'items'") + +# These are the actual classes you use! +class COWDictBase(object): + __metaclass__ = COWDictMeta + __count__ = 0 + +class COWSetBase(object): + __metaclass__ = COWSetMeta + __count__ = 0 + +if __name__ == "__main__": + import sys + COWDictBase.__warn__ = sys.stderr + a = COWDictBase() + print "a", a + + a['a'] = 'a' + a['b'] = 'b' + a['dict'] = {} + + b = a.copy() + print "b", b + b['c'] = 'b' + + print + + print "a", a + for x in a.iteritems(): + print x + print "--" + print "b", b + for x in b.iteritems(): + print x + print + + b['dict']['a'] = 'b' + b['a'] = 'c' + + print "a", a + for x in a.iteritems(): + print x + print "--" + print "b", b + for x in b.iteritems(): + print x + print + + try: + b['dict2'] + except KeyError, e: + print "Okay!" + + a['set'] = COWSetBase() + a['set'].add("o1") + a['set'].add("o1") + a['set'].add("o2") + + print "a", a + for x in a['set'].itervalues(): + print x + print "--" + print "b", b + for x in b['set'].itervalues(): + print x + print + + b['set'].add('o3') + + print "a", a + for x in a['set'].itervalues(): + print x + print "--" + print "b", b + for x in b['set'].itervalues(): + print x + print + + a['set2'] = set() + a['set2'].add("o1") + a['set2'].add("o1") + a['set2'].add("o2") + + print "a", a + for x in a.iteritems(): + print x + print "--" + print "b", b + for x in b.iteritems(readonly=True): + print x + print + + del b['b'] + try: + print b['b'] + except KeyError: + print "Yay! deleted key raises error" + + if b.has_key('b'): + print "Boo!" + else: + print "Yay - has_key with delete works!" + + print "a", a + for x in a.iteritems(): + print x + print "--" + print "b", b + for x in b.iteritems(readonly=True): + print x + print + + b.__revertitem__('b') + + print "a", a + for x in a.iteritems(): + print x + print "--" + print "b", b + for x in b.iteritems(readonly=True): + print x + print + + b.__revertitem__('dict') + print "a", a + for x in a.iteritems(): + print x + print "--" + print "b", b + for x in b.iteritems(readonly=True): + print x + print diff --git a/bitbake-dev/lib/bb/__init__.py b/bitbake-dev/lib/bb/__init__.py new file mode 100644 index 0000000000..99995212c3 --- /dev/null +++ b/bitbake-dev/lib/bb/__init__.py @@ -0,0 +1,1133 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Build System Python Library +# +# Copyright (C) 2003 Holger Schurig +# Copyright (C) 2003, 2004 Chris Larson +# +# Based on Gentoo's portage.py. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +__version__ = "1.9.0" + +__all__ = [ + + "debug", + "note", + "error", + "fatal", + + "mkdirhier", + "movefile", + + "tokenize", + "evaluate", + "flatten", + "relparse", + "ververify", + "isjustname", + "isspecific", + "pkgsplit", + "catpkgsplit", + "vercmp", + "pkgcmp", + "dep_parenreduce", + "dep_opconvert", + +# fetch + "decodeurl", + "encodeurl", + +# modules + "parse", + "data", + "command", + "event", + "build", + "fetch", + "manifest", + "methodpool", + "cache", + "runqueue", + "taskdata", + "providers", + ] + +whitespace = '\t\n\x0b\x0c\r ' +lowercase = 'abcdefghijklmnopqrstuvwxyz' + +import sys, os, types, re, string, bb +from bb import msg + +#projectdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) +projectdir = os.getcwd() + +if "BBDEBUG" in os.environ: + level = int(os.environ["BBDEBUG"]) + if level: + bb.msg.set_debug_level(level) + +class VarExpandError(Exception): + pass + +class MalformedUrl(Exception): + """Exception raised when encountering an invalid url""" + + +####################################################################### +####################################################################### +# +# SECTION: Debug +# +# PURPOSE: little functions to make yourself known +# +####################################################################### +####################################################################### + +def plain(*args): + bb.msg.warn(''.join(args)) + +def debug(lvl, *args): + bb.msg.debug(lvl, None, ''.join(args)) + +def note(*args): + bb.msg.note(1, None, ''.join(args)) + +def warn(*args): + bb.msg.warn(1, None, ''.join(args)) + +def error(*args): + bb.msg.error(None, ''.join(args)) + +def fatal(*args): + bb.msg.fatal(None, ''.join(args)) + + +####################################################################### +####################################################################### +# +# SECTION: File +# +# PURPOSE: Basic file and directory tree related functions +# +####################################################################### +####################################################################### + +def mkdirhier(dir): + """Create a directory like 'mkdir -p', but does not complain if + directory already exists like os.makedirs + """ + + debug(3, "mkdirhier(%s)" % dir) + try: + os.makedirs(dir) + debug(2, "created " + dir) + except OSError, e: + if e.errno != 17: raise e + + +####################################################################### + +import stat + +def movefile(src,dest,newmtime=None,sstat=None): + """Moves a file from src to dest, preserving all permissions and + attributes; mtime will be preserved even when moving across + filesystems. Returns true on success and false on failure. Move is + atomic. + """ + + #print "movefile("+src+","+dest+","+str(newmtime)+","+str(sstat)+")" + try: + if not sstat: + sstat=os.lstat(src) + except Exception, e: + print "movefile: Stating source file failed...", e + return None + + destexists=1 + try: + dstat=os.lstat(dest) + except: + dstat=os.lstat(os.path.dirname(dest)) + destexists=0 + + if destexists: + if stat.S_ISLNK(dstat[stat.ST_MODE]): + try: + os.unlink(dest) + destexists=0 + except Exception, e: + pass + + if stat.S_ISLNK(sstat[stat.ST_MODE]): + try: + target=os.readlink(src) + if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): + os.unlink(dest) + os.symlink(target,dest) + #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) + os.unlink(src) + return os.lstat(dest) + except Exception, e: + print "movefile: failed to properly create symlink:", dest, "->", target, e + return None + + renamefailed=1 + if sstat[stat.ST_DEV]==dstat[stat.ST_DEV]: + try: + ret=os.rename(src,dest) + renamefailed=0 + except Exception, e: + import errno + if e[0]!=errno.EXDEV: + # Some random error. + print "movefile: Failed to move", src, "to", dest, e + return None + # Invalid cross-device-link 'bind' mounted or actually Cross-Device + + if renamefailed: + didcopy=0 + if stat.S_ISREG(sstat[stat.ST_MODE]): + try: # For safety copy then move it over. + shutil.copyfile(src,dest+"#new") + os.rename(dest+"#new",dest) + didcopy=1 + except Exception, e: + print 'movefile: copy', src, '->', dest, 'failed.', e + return None + else: + #we don't yet handle special, so we need to fall back to /bin/mv + a=getstatusoutput("/bin/mv -f "+"'"+src+"' '"+dest+"'") + if a[0]!=0: + print "movefile: Failed to move special file:" + src + "' to '" + dest + "'", a + return None # failure + try: + if didcopy: + missingos.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) + os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown + os.unlink(src) + except Exception, e: + print "movefile: Failed to chown/chmod/unlink", dest, e + return None + + if newmtime: + os.utime(dest,(newmtime,newmtime)) + else: + os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME])) + newmtime=sstat[stat.ST_MTIME] + return newmtime + +def copyfile(src,dest,newmtime=None,sstat=None): + """ + Copies a file from src to dest, preserving all permissions and + attributes; mtime will be preserved even when moving across + filesystems. Returns true on success and false on failure. + """ + import os, stat, shutil + + #print "copyfile("+src+","+dest+","+str(newmtime)+","+str(sstat)+")" + try: + if not sstat: + sstat=os.lstat(src) + except Exception, e: + print "copyfile: Stating source file failed...", e + return False + + destexists=1 + try: + dstat=os.lstat(dest) + except: + dstat=os.lstat(os.path.dirname(dest)) + destexists=0 + + if destexists: + if stat.S_ISLNK(dstat[stat.ST_MODE]): + try: + os.unlink(dest) + destexists=0 + except Exception, e: + pass + + if stat.S_ISLNK(sstat[stat.ST_MODE]): + try: + target=os.readlink(src) + if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): + os.unlink(dest) + os.symlink(target,dest) + #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) + return os.lstat(dest) + except Exception, e: + print "copyfile: failed to properly create symlink:", dest, "->", target, e + return False + + if stat.S_ISREG(sstat[stat.ST_MODE]): + try: # For safety copy then move it over. + shutil.copyfile(src,dest+"#new") + os.rename(dest+"#new",dest) + except Exception, e: + print 'copyfile: copy', src, '->', dest, 'failed.', e + return False + else: + #we don't yet handle special, so we need to fall back to /bin/mv + a=getstatusoutput("/bin/cp -f "+"'"+src+"' '"+dest+"'") + if a[0]!=0: + print "copyfile: Failed to copy special file:" + src + "' to '" + dest + "'", a + return False # failure + try: + os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) + os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown + except Exception, e: + print "copyfile: Failed to chown/chmod/unlink", dest, e + return False + + if newmtime: + os.utime(dest,(newmtime,newmtime)) + else: + os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME])) + newmtime=sstat[stat.ST_MTIME] + return newmtime + +####################################################################### +####################################################################### +# +# SECTION: Download +# +# PURPOSE: Download via HTTP, FTP, CVS, BITKEEPER, handling of MD5-signatures +# and mirrors +# +####################################################################### +####################################################################### + +def decodeurl(url): + """Decodes an URL into the tokens (scheme, network location, path, + user, password, parameters). + + >>> decodeurl("http://www.google.com/index.html") + ('http', 'www.google.com', '/index.html', '', '', {}) + + CVS url with username, host and cvsroot. The cvs module to check out is in the + parameters: + + >>> decodeurl("cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg") + ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}) + + Dito, but this time the username has a password part. And we also request a special tag + to check out. + + >>> decodeurl("cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81") + ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}) + """ + + m = re.compile('(?P[^:]*)://((?P.+)@)?(?P[^;]+)(;(?P.*))?').match(url) + if not m: + raise MalformedUrl(url) + + type = m.group('type') + location = m.group('location') + if not location: + raise MalformedUrl(url) + user = m.group('user') + parm = m.group('parm') + + locidx = location.find('/') + if locidx != -1: + host = location[:locidx] + path = location[locidx:] + else: + host = "" + path = location + if user: + m = re.compile('(?P[^:]+)(:?(?P.*))').match(user) + if m: + user = m.group('user') + pswd = m.group('pswd') + else: + user = '' + pswd = '' + + p = {} + if parm: + for s in parm.split(';'): + s1,s2 = s.split('=') + p[s1] = s2 + + return (type, host, path, user, pswd, p) + +####################################################################### + +def encodeurl(decoded): + """Encodes a URL from tokens (scheme, network location, path, + user, password, parameters). + + >>> encodeurl(['http', 'www.google.com', '/index.html', '', '', {}]) + 'http://www.google.com/index.html' + + CVS with username, host and cvsroot. The cvs module to check out is in the + parameters: + + >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}]) + 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg' + + Dito, but this time the username has a password part. And we also request a special tag + to check out. + + >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}]) + 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg' + """ + + (type, host, path, user, pswd, p) = decoded + + if not type or not path: + fatal("invalid or missing parameters for url encoding") + url = '%s://' % type + if user: + url += "%s" % user + if pswd: + url += ":%s" % pswd + url += "@" + if host: + url += "%s" % host + url += "%s" % path + if p: + for parm in p.keys(): + url += ";%s=%s" % (parm, p[parm]) + + return url + +####################################################################### + +def which(path, item, direction = 0): + """ + Locate a file in a PATH + """ + + paths = (path or "").split(':') + if direction != 0: + paths.reverse() + + for p in (path or "").split(':'): + next = os.path.join(p, item) + if os.path.exists(next): + return next + + return "" + +####################################################################### + + + + +####################################################################### +####################################################################### +# +# SECTION: Dependency +# +# PURPOSE: Compare build & run dependencies +# +####################################################################### +####################################################################### + +def tokenize(mystring): + """Breaks a string like 'foo? (bar) oni? (blah (blah))' into (possibly embedded) lists: + + >>> tokenize("x") + ['x'] + >>> tokenize("x y") + ['x', 'y'] + >>> tokenize("(x y)") + [['x', 'y']] + >>> tokenize("(x y) b c") + [['x', 'y'], 'b', 'c'] + >>> tokenize("foo? (bar) oni? (blah (blah))") + ['foo?', ['bar'], 'oni?', ['blah', ['blah']]] + >>> tokenize("sys-apps/linux-headers nls? (sys-devel/gettext)") + ['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']] + """ + + newtokens = [] + curlist = newtokens + prevlists = [] + level = 0 + accum = "" + for x in mystring: + if x=="(": + if accum: + curlist.append(accum) + accum="" + prevlists.append(curlist) + curlist=[] + level=level+1 + elif x==")": + if accum: + curlist.append(accum) + accum="" + if level==0: + print "!!! tokenizer: Unmatched left parenthesis in:\n'"+mystring+"'" + return None + newlist=curlist + curlist=prevlists.pop() + curlist.append(newlist) + level=level-1 + elif x in whitespace: + if accum: + curlist.append(accum) + accum="" + else: + accum=accum+x + if accum: + curlist.append(accum) + if (level!=0): + print "!!! tokenizer: Exiting with unterminated parenthesis in:\n'"+mystring+"'" + return None + return newtokens + + +####################################################################### + +def evaluate(tokens,mydefines,allon=0): + """Removes tokens based on whether conditional definitions exist or not. + Recognizes ! + + >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {}) + ['sys-apps/linux-headers'] + + Negate the flag: + + >>> evaluate(['sys-apps/linux-headers', '!nls?', ['sys-devel/gettext']], {}) + ['sys-apps/linux-headers', ['sys-devel/gettext']] + + Define 'nls': + + >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {"nls":1}) + ['sys-apps/linux-headers', ['sys-devel/gettext']] + + Turn allon on: + + >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {}, True) + ['sys-apps/linux-headers', ['sys-devel/gettext']] + """ + + if tokens == None: + return None + mytokens = tokens + [] # this copies the list + pos = 0 + while pos < len(mytokens): + if type(mytokens[pos]) == types.ListType: + evaluate(mytokens[pos], mydefines) + if not len(mytokens[pos]): + del mytokens[pos] + continue + elif mytokens[pos][-1] == "?": + cur = mytokens[pos][:-1] + del mytokens[pos] + if allon: + if cur[0] == "!": + del mytokens[pos] + else: + if cur[0] == "!": + if (cur[1:] in mydefines) and (pos < len(mytokens)): + del mytokens[pos] + continue + elif (cur not in mydefines) and (pos < len(mytokens)): + del mytokens[pos] + continue + pos = pos + 1 + return mytokens + + +####################################################################### + +def flatten(mytokens): + """Converts nested arrays into a flat arrays: + + >>> flatten([1,[2,3]]) + [1, 2, 3] + >>> flatten(['sys-apps/linux-headers', ['sys-devel/gettext']]) + ['sys-apps/linux-headers', 'sys-devel/gettext'] + """ + + newlist=[] + for x in mytokens: + if type(x)==types.ListType: + newlist.extend(flatten(x)) + else: + newlist.append(x) + return newlist + + +####################################################################### + +_package_weights_ = {"pre":-2,"p":0,"alpha":-4,"beta":-3,"rc":-1} # dicts are unordered +_package_ends_ = ["pre", "p", "alpha", "beta", "rc", "cvs", "bk", "HEAD" ] # so we need ordered list + +def relparse(myver): + """Parses the last elements of a version number into a triplet, that can + later be compared: + + >>> relparse('1.2_pre3') + [1.2, -2, 3.0] + >>> relparse('1.2b') + [1.2, 98, 0] + >>> relparse('1.2') + [1.2, 0, 0] + """ + + number = 0 + p1 = 0 + p2 = 0 + mynewver = myver.split('_') + if len(mynewver)==2: + # an _package_weights_ + number = float(mynewver[0]) + match = 0 + for x in _package_ends_: + elen = len(x) + if mynewver[1][:elen] == x: + match = 1 + p1 = _package_weights_[x] + try: + p2 = float(mynewver[1][elen:]) + except: + p2 = 0 + break + if not match: + # normal number or number with letter at end + divider = len(myver)-1 + if myver[divider:] not in "1234567890": + # letter at end + p1 = ord(myver[divider:]) + number = float(myver[0:divider]) + else: + number = float(myver) + else: + # normal number or number with letter at end + divider = len(myver)-1 + if myver[divider:] not in "1234567890": + #letter at end + p1 = ord(myver[divider:]) + number = float(myver[0:divider]) + else: + number = float(myver) + return [number,p1,p2] + + +####################################################################### + +__ververify_cache__ = {} + +def ververify(myorigval,silent=1): + """Returns 1 if given a valid version string, els 0. Valid versions are in the format + + ....[a-z,_{_package_weights_}[vy]] + + >>> ververify('2.4.20') + 1 + >>> ververify('2.4..20') # two dots + 0 + >>> ververify('2.x.20') # 'x' is not numeric + 0 + >>> ververify('2.4.20a') + 1 + >>> ververify('2.4.20cvs') # only one trailing letter + 0 + >>> ververify('1a') + 1 + >>> ververify('test_a') # no version at all + 0 + >>> ververify('2.4.20_beta1') + 1 + >>> ververify('2.4.20_beta') + 1 + >>> ververify('2.4.20_wrongext') # _wrongext is no valid trailer + 0 + """ + + # Lookup the cache first + try: + return __ververify_cache__[myorigval] + except KeyError: + pass + + if len(myorigval) == 0: + if not silent: + error("package version is empty") + __ververify_cache__[myorigval] = 0 + return 0 + myval = myorigval.split('.') + if len(myval)==0: + if not silent: + error("package name has empty version string") + __ververify_cache__[myorigval] = 0 + return 0 + # all but the last version must be a numeric + for x in myval[:-1]: + if not len(x): + if not silent: + error("package version has two points in a row") + __ververify_cache__[myorigval] = 0 + return 0 + try: + foo = int(x) + except: + if not silent: + error("package version contains non-numeric '"+x+"'") + __ververify_cache__[myorigval] = 0 + return 0 + if not len(myval[-1]): + if not silent: + error("package version has trailing dot") + __ververify_cache__[myorigval] = 0 + return 0 + try: + foo = int(myval[-1]) + __ververify_cache__[myorigval] = 1 + return 1 + except: + pass + + # ok, our last component is not a plain number or blank, let's continue + if myval[-1][-1] in lowercase: + try: + foo = int(myval[-1][:-1]) + return 1 + __ververify_cache__[myorigval] = 1 + # 1a, 2.0b, etc. + except: + pass + # ok, maybe we have a 1_alpha or 1_beta2; let's see + ep=string.split(myval[-1],"_") + if len(ep)!= 2: + if not silent: + error("package version has more than one letter at then end") + __ververify_cache__[myorigval] = 0 + return 0 + try: + foo = string.atoi(ep[0]) + except: + # this needs to be numeric, i.e. the "1" in "1_alpha" + if not silent: + error("package version must have numeric part before the '_'") + __ververify_cache__[myorigval] = 0 + return 0 + + for mye in _package_ends_: + if ep[1][0:len(mye)] == mye: + if len(mye) == len(ep[1]): + # no trailing numeric is ok + __ververify_cache__[myorigval] = 1 + return 1 + else: + try: + foo = string.atoi(ep[1][len(mye):]) + __ververify_cache__[myorigval] = 1 + return 1 + except: + # if no _package_weights_ work, *then* we return 0 + pass + if not silent: + error("package version extension after '_' is invalid") + __ververify_cache__[myorigval] = 0 + return 0 + + +def isjustname(mypkg): + myparts = string.split(mypkg,'-') + for x in myparts: + if ververify(x): + return 0 + return 1 + + +_isspecific_cache_={} + +def isspecific(mypkg): + "now supports packages with no category" + try: + return __isspecific_cache__[mypkg] + except: + pass + + mysplit = string.split(mypkg,"/") + if not isjustname(mysplit[-1]): + __isspecific_cache__[mypkg] = 1 + return 1 + __isspecific_cache__[mypkg] = 0 + return 0 + + +####################################################################### + +__pkgsplit_cache__={} + +def pkgsplit(mypkg, silent=1): + + """This function can be used as a package verification function. If + it is a valid name, pkgsplit will return a list containing: + [pkgname, pkgversion(norev), pkgrev ]. + + >>> pkgsplit('') + >>> pkgsplit('x') + >>> pkgsplit('x-') + >>> pkgsplit('-1') + >>> pkgsplit('glibc-1.2-8.9-r7') + >>> pkgsplit('glibc-2.2.5-r7') + ['glibc', '2.2.5', 'r7'] + >>> pkgsplit('foo-1.2-1') + >>> pkgsplit('Mesa-3.0') + ['Mesa', '3.0', 'r0'] + """ + + try: + return __pkgsplit_cache__[mypkg] + except KeyError: + pass + + myparts = string.split(mypkg,'-') + if len(myparts) < 2: + if not silent: + error("package name without name or version part") + __pkgsplit_cache__[mypkg] = None + return None + for x in myparts: + if len(x) == 0: + if not silent: + error("package name with empty name or version part") + __pkgsplit_cache__[mypkg] = None + return None + # verify rev + revok = 0 + myrev = myparts[-1] + ververify(myrev, silent) + if len(myrev) and myrev[0] == "r": + try: + string.atoi(myrev[1:]) + revok = 1 + except: + pass + if revok: + if ververify(myparts[-2]): + if len(myparts) == 2: + __pkgsplit_cache__[mypkg] = None + return None + else: + for x in myparts[:-2]: + if ververify(x): + __pkgsplit_cache__[mypkg]=None + return None + # names can't have versiony looking parts + myval=[string.join(myparts[:-2],"-"),myparts[-2],myparts[-1]] + __pkgsplit_cache__[mypkg]=myval + return myval + else: + __pkgsplit_cache__[mypkg] = None + return None + + elif ververify(myparts[-1],silent): + if len(myparts)==1: + if not silent: + print "!!! Name error in",mypkg+": missing name part." + __pkgsplit_cache__[mypkg]=None + return None + else: + for x in myparts[:-1]: + if ververify(x): + if not silent: error("package name has multiple version parts") + __pkgsplit_cache__[mypkg] = None + return None + myval = [string.join(myparts[:-1],"-"), myparts[-1],"r0"] + __pkgsplit_cache__[mypkg] = myval + return myval + else: + __pkgsplit_cache__[mypkg] = None + return None + + +####################################################################### + +__catpkgsplit_cache__ = {} + +def catpkgsplit(mydata,silent=1): + """returns [cat, pkgname, version, rev ] + + >>> catpkgsplit('sys-libs/glibc-1.2-r7') + ['sys-libs', 'glibc', '1.2', 'r7'] + >>> catpkgsplit('glibc-1.2-r7') + [None, 'glibc', '1.2', 'r7'] + """ + + try: + return __catpkgsplit_cache__[mydata] + except KeyError: + pass + + cat = os.path.basename(os.path.dirname(mydata)) + mydata = os.path.join(cat, os.path.basename(mydata)) + if mydata[-3:] == '.bb': + mydata = mydata[:-3] + + mysplit = mydata.split("/") + p_split = None + splitlen = len(mysplit) + if splitlen == 1: + retval = [None] + p_split = pkgsplit(mydata,silent) + else: + retval = [mysplit[splitlen - 2]] + p_split = pkgsplit(mysplit[splitlen - 1],silent) + if not p_split: + __catpkgsplit_cache__[mydata] = None + return None + retval.extend(p_split) + __catpkgsplit_cache__[mydata] = retval + return retval + + +####################################################################### + +__vercmp_cache__ = {} + +def vercmp(val1,val2): + """This takes two version strings and returns an integer to tell you whether + the versions are the same, val1>val2 or val2>val1. + + >>> vercmp('1', '2') + -1.0 + >>> vercmp('2', '1') + 1.0 + >>> vercmp('1', '1.0') + 0 + >>> vercmp('1', '1.1') + -1.0 + >>> vercmp('1.1', '1_p2') + 1.0 + """ + + # quick short-circuit + if val1 == val2: + return 0 + valkey = val1+" "+val2 + + # cache lookup + try: + return __vercmp_cache__[valkey] + try: + return - __vercmp_cache__[val2+" "+val1] + except KeyError: + pass + except KeyError: + pass + + # consider 1_p2 vc 1.1 + # after expansion will become (1_p2,0) vc (1,1) + # then 1_p2 is compared with 1 before 0 is compared with 1 + # to solve the bug we need to convert it to (1,0_p2) + # by splitting _prepart part and adding it back _after_expansion + + val1_prepart = val2_prepart = '' + if val1.count('_'): + val1, val1_prepart = val1.split('_', 1) + if val2.count('_'): + val2, val2_prepart = val2.split('_', 1) + + # replace '-' by '.' + # FIXME: Is it needed? can val1/2 contain '-'? + + val1 = string.split(val1,'-') + if len(val1) == 2: + val1[0] = val1[0] +"."+ val1[1] + val2 = string.split(val2,'-') + if len(val2) == 2: + val2[0] = val2[0] +"."+ val2[1] + + val1 = string.split(val1[0],'.') + val2 = string.split(val2[0],'.') + + # add back decimal point so that .03 does not become "3" ! + for x in range(1,len(val1)): + if val1[x][0] == '0' : + val1[x] = '.' + val1[x] + for x in range(1,len(val2)): + if val2[x][0] == '0' : + val2[x] = '.' + val2[x] + + # extend varion numbers + if len(val2) < len(val1): + val2.extend(["0"]*(len(val1)-len(val2))) + elif len(val1) < len(val2): + val1.extend(["0"]*(len(val2)-len(val1))) + + # add back _prepart tails + if val1_prepart: + val1[-1] += '_' + val1_prepart + if val2_prepart: + val2[-1] += '_' + val2_prepart + # The above code will extend version numbers out so they + # have the same number of digits. + for x in range(0,len(val1)): + cmp1 = relparse(val1[x]) + cmp2 = relparse(val2[x]) + for y in range(0,3): + myret = cmp1[y] - cmp2[y] + if myret != 0: + __vercmp_cache__[valkey] = myret + return myret + __vercmp_cache__[valkey] = 0 + return 0 + + +####################################################################### + +def pkgcmp(pkg1,pkg2): + """ Compares two packages, which should have been split via + pkgsplit(). if the return value val is less than zero, then pkg2 is + newer than pkg1, zero if equal and positive if older. + + >>> pkgcmp(['glibc', '2.2.5', 'r7'], ['glibc', '2.2.5', 'r7']) + 0 + >>> pkgcmp(['glibc', '2.2.5', 'r4'], ['glibc', '2.2.5', 'r7']) + -1 + >>> pkgcmp(['glibc', '2.2.5', 'r7'], ['glibc', '2.2.5', 'r2']) + 1 + """ + + mycmp = vercmp(pkg1[1],pkg2[1]) + if mycmp > 0: + return 1 + if mycmp < 0: + return -1 + r1=string.atoi(pkg1[2][1:]) + r2=string.atoi(pkg2[2][1:]) + if r1 > r2: + return 1 + if r2 > r1: + return -1 + return 0 + + +####################################################################### + +def dep_parenreduce(mysplit, mypos=0): + """Accepts a list of strings, and converts '(' and ')' surrounded items to sub-lists: + + >>> dep_parenreduce(['']) + [''] + >>> dep_parenreduce(['1', '2', '3']) + ['1', '2', '3'] + >>> dep_parenreduce(['1', '(', '2', '3', ')', '4']) + ['1', ['2', '3'], '4'] + """ + + while mypos < len(mysplit): + if mysplit[mypos] == "(": + firstpos = mypos + mypos = mypos + 1 + while mypos < len(mysplit): + if mysplit[mypos] == ")": + mysplit[firstpos:mypos+1] = [mysplit[firstpos+1:mypos]] + mypos = firstpos + break + elif mysplit[mypos] == "(": + # recurse + mysplit = dep_parenreduce(mysplit,mypos) + mypos = mypos + 1 + mypos = mypos + 1 + return mysplit + + +def dep_opconvert(mysplit, myuse): + "Does dependency operator conversion" + + mypos = 0 + newsplit = [] + while mypos < len(mysplit): + if type(mysplit[mypos]) == types.ListType: + newsplit.append(dep_opconvert(mysplit[mypos],myuse)) + mypos += 1 + elif mysplit[mypos] == ")": + # mismatched paren, error + return None + elif mysplit[mypos]=="||": + if ((mypos+1)>=len(mysplit)) or (type(mysplit[mypos+1])!=types.ListType): + # || must be followed by paren'd list + return None + try: + mynew = dep_opconvert(mysplit[mypos+1],myuse) + except Exception, e: + error("unable to satisfy OR dependancy: " + string.join(mysplit," || ")) + raise e + mynew[0:0] = ["||"] + newsplit.append(mynew) + mypos += 2 + elif mysplit[mypos][-1] == "?": + # use clause, i.e "gnome? ( foo bar )" + # this is a quick and dirty hack so that repoman can enable all USE vars: + if (len(myuse) == 1) and (myuse[0] == "*"): + # enable it even if it's ! (for repoman) but kill it if it's + # an arch variable that isn't for this arch. XXX Sparc64? + if (mysplit[mypos][:-1] not in settings.usemask) or \ + (mysplit[mypos][:-1]==settings["ARCH"]): + enabled=1 + else: + enabled=0 + else: + if mysplit[mypos][0] == "!": + myusevar = mysplit[mypos][1:-1] + enabled = not myusevar in myuse + #if myusevar in myuse: + # enabled = 0 + #else: + # enabled = 1 + else: + myusevar=mysplit[mypos][:-1] + enabled = myusevar in myuse + #if myusevar in myuse: + # enabled=1 + #else: + # enabled=0 + if (mypos +2 < len(mysplit)) and (mysplit[mypos+2] == ":"): + # colon mode + if enabled: + # choose the first option + if type(mysplit[mypos+1]) == types.ListType: + newsplit.append(dep_opconvert(mysplit[mypos+1],myuse)) + else: + newsplit.append(mysplit[mypos+1]) + else: + # choose the alternate option + if type(mysplit[mypos+1]) == types.ListType: + newsplit.append(dep_opconvert(mysplit[mypos+3],myuse)) + else: + newsplit.append(mysplit[mypos+3]) + mypos += 4 + else: + # normal use mode + if enabled: + if type(mysplit[mypos+1]) == types.ListType: + newsplit.append(dep_opconvert(mysplit[mypos+1],myuse)) + else: + newsplit.append(mysplit[mypos+1]) + # otherwise, continue + mypos += 2 + else: + # normal item + newsplit.append(mysplit[mypos]) + mypos += 1 + return newsplit + +if __name__ == "__main__": + import doctest, bb + doctest.testmod(bb) diff --git a/bitbake-dev/lib/bb/build.py b/bitbake-dev/lib/bb/build.py new file mode 100644 index 0000000000..ca7cfbc6bb --- /dev/null +++ b/bitbake-dev/lib/bb/build.py @@ -0,0 +1,377 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake 'Build' implementation +# +# Core code for function execution and task handling in the +# BitBake build tools. +# +# Copyright (C) 2003, 2004 Chris Larson +# +# Based on Gentoo's portage.py. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +#Based on functions from the base bb module, Copyright 2003 Holger Schurig + +from bb import data, event, mkdirhier, utils +import bb, os, sys + +# events +class FuncFailed(Exception): + """ + Executed function failed + First parameter a message + Second paramter is a logfile (optional) + """ + +class EventException(Exception): + """Exception which is associated with an Event.""" + + def __init__(self, msg, event): + self.args = msg, event + +class TaskBase(event.Event): + """Base class for task events""" + + def __init__(self, t, d ): + self._task = t + self._package = bb.data.getVar("PF", d, 1) + event.Event.__init__(self, d) + self._message = "package %s: task %s: %s" % (bb.data.getVar("PF", d, 1), t, bb.event.getName(self)[4:]) + + def getTask(self): + return self._task + + def setTask(self, task): + self._task = task + + task = property(getTask, setTask, None, "task property") + +class TaskStarted(TaskBase): + """Task execution started""" + +class TaskSucceeded(TaskBase): + """Task execution completed""" + +class TaskFailed(TaskBase): + """Task execution failed""" + def __init__(self, msg, logfile, t, d ): + self.logfile = logfile + self.msg = msg + TaskBase.__init__(self, t, d) + +class InvalidTask(TaskBase): + """Invalid Task""" + +# functions + +def exec_func(func, d, dirs = None): + """Execute an BB 'function'""" + + body = data.getVar(func, d) + if not body: + return + + flags = data.getVarFlags(func, d) + for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot']: + if not item in flags: + flags[item] = None + + ispython = flags['python'] + + cleandirs = (data.expand(flags['cleandirs'], d) or "").split() + for cdir in cleandirs: + os.system("rm -rf %s" % cdir) + + if dirs: + dirs = data.expand(dirs, d) + else: + dirs = (data.expand(flags['dirs'], d) or "").split() + for adir in dirs: + mkdirhier(adir) + + if len(dirs) > 0: + adir = dirs[-1] + else: + adir = data.getVar('B', d, 1) + + # Save current directory + try: + prevdir = os.getcwd() + except OSError: + prevdir = data.getVar('TOPDIR', d, True) + + # Setup logfiles + t = data.getVar('T', d, 1) + if not t: + bb.msg.fatal(bb.msg.domain.Build, "T not set") + mkdirhier(t) + # Gross hack, FIXME + import random + logfile = "%s/log.%s.%s.%s" % (t, func, str(os.getpid()),random.random()) + runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) + + # Change to correct directory (if specified) + if adir and os.access(adir, os.F_OK): + os.chdir(adir) + + # Handle logfiles + si = file('/dev/null', 'r') + try: + if bb.msg.debug_level['default'] > 0 or ispython: + so = os.popen("tee \"%s\"" % logfile, "w") + else: + so = file(logfile, 'w') + except OSError, e: + bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e) + pass + + se = so + + # Dup the existing fds so we dont lose them + osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] + oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] + ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] + + # Replace those fds with our own + os.dup2(si.fileno(), osi[1]) + os.dup2(so.fileno(), oso[1]) + os.dup2(se.fileno(), ose[1]) + + locks = [] + lockfiles = (data.expand(flags['lockfiles'], d) or "").split() + for lock in lockfiles: + locks.append(bb.utils.lockfile(lock)) + + try: + # Run the function + if ispython: + exec_func_python(func, d, runfile, logfile) + else: + exec_func_shell(func, d, runfile, logfile, flags) + + # Restore original directory + try: + os.chdir(prevdir) + except: + pass + + finally: + + # Unlock any lockfiles + for lock in locks: + bb.utils.unlockfile(lock) + + # Restore the backup fds + os.dup2(osi[0], osi[1]) + os.dup2(oso[0], oso[1]) + os.dup2(ose[0], ose[1]) + + # Close our logs + si.close() + so.close() + se.close() + + # Close the backup fds + os.close(osi[0]) + os.close(oso[0]) + os.close(ose[0]) + +def exec_func_python(func, d, runfile, logfile): + """Execute a python BB 'function'""" + import re, os + + bbfile = bb.data.getVar('FILE', d, 1) + tmp = "def " + func + "():\n%s" % data.getVar(func, d) + tmp += '\n' + func + '()' + + f = open(runfile, "w") + f.write(tmp) + comp = utils.better_compile(tmp, func, bbfile) + g = {} # globals + g['bb'] = bb + g['os'] = os + g['d'] = d + utils.better_exec(comp, g, tmp, bbfile) + + +def exec_func_shell(func, d, runfile, logfile, flags): + """Execute a shell BB 'function' Returns true if execution was successful. + + For this, it creates a bash shell script in the tmp dectory, writes the local + data into it and finally executes. The output of the shell will end in a log file and stdout. + + Note on directory behavior. The 'dirs' varflag should contain a list + of the directories you need created prior to execution. The last + item in the list is where we will chdir/cd to. + """ + + deps = flags['deps'] + check = flags['check'] + if check in globals(): + if globals()[check](func, deps): + return + + f = open(runfile, "w") + f.write("#!/bin/sh -e\n") + if bb.msg.debug_level['default'] > 0: f.write("set -x\n") + data.emit_env(f, d) + + f.write("cd %s\n" % os.getcwd()) + if func: f.write("%s\n" % func) + f.close() + os.chmod(runfile, 0775) + if not func: + bb.msg.error(bb.msg.domain.Build, "Function not specified") + raise FuncFailed("Function not specified for exec_func_shell") + + # execute function + if flags['fakeroot']: + maybe_fakeroot = "PATH=\"%s\" fakeroot " % bb.data.getVar("PATH", d, 1) + else: + maybe_fakeroot = '' + lang_environment = "LC_ALL=C " + ret = os.system('%s%ssh -e %s' % (lang_environment, maybe_fakeroot, runfile)) + + if ret == 0: + return + + bb.msg.error(bb.msg.domain.Build, "Function %s failed" % func) + raise FuncFailed("function %s failed" % func, logfile) + + +def exec_task(task, d): + """Execute an BB 'task' + + The primary difference between executing a task versus executing + a function is that a task exists in the task digraph, and therefore + has dependencies amongst other tasks.""" + + # Check whther this is a valid task + if not data.getVarFlag(task, 'task', d): + raise EventException("No such task", InvalidTask(task, d)) + + try: + bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % task) + old_overrides = data.getVar('OVERRIDES', d, 0) + localdata = data.createCopy(d) + data.setVar('OVERRIDES', 'task_%s:%s' % (task, old_overrides), localdata) + data.update_data(localdata) + event.fire(TaskStarted(task, localdata)) + exec_func(task, localdata) + event.fire(TaskSucceeded(task, localdata)) + except FuncFailed, message: + # Try to extract the optional logfile + try: + (msg, logfile) = message + except: + logfile = None + msg = message + bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % message ) + failedevent = TaskFailed(msg, logfile, task, d) + event.fire(failedevent) + raise EventException("Function failed in task: %s" % message, failedevent) + + # make stamp, or cause event and raise exception + if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d): + make_stamp(task, d) + +def extract_stamp(d, fn): + """ + Extracts stamp format which is either a data dictonary (fn unset) + or a dataCache entry (fn set). + """ + if fn: + return d.stamp[fn] + return data.getVar('STAMP', d, 1) + +def stamp_internal(task, d, file_name): + """ + Internal stamp helper function + Removes any stamp for the given task + Makes sure the stamp directory exists + Returns the stamp path+filename + """ + stamp = extract_stamp(d, file_name) + if not stamp: + return + stamp = "%s.%s" % (stamp, task) + mkdirhier(os.path.dirname(stamp)) + # Remove the file and recreate to force timestamp + # change on broken NFS filesystems + if os.access(stamp, os.F_OK): + os.remove(stamp) + return stamp + +def make_stamp(task, d, file_name = None): + """ + Creates/updates a stamp for a given task + (d can be a data dict or dataCache) + """ + stamp = stamp_internal(task, d, file_name) + if stamp: + f = open(stamp, "w") + f.close() + +def del_stamp(task, d, file_name = None): + """ + Removes a stamp for a given task + (d can be a data dict or dataCache) + """ + stamp_internal(task, d, file_name) + +def add_tasks(tasklist, d): + task_deps = data.getVar('_task_deps', d) + if not task_deps: + task_deps = {} + if not 'tasks' in task_deps: + task_deps['tasks'] = [] + if not 'parents' in task_deps: + task_deps['parents'] = {} + + for task in tasklist: + task = data.expand(task, d) + data.setVarFlag(task, 'task', 1, d) + + if not task in task_deps['tasks']: + task_deps['tasks'].append(task) + + flags = data.getVarFlags(task, d) + def getTask(name): + if not name in task_deps: + task_deps[name] = {} + if name in flags: + deptask = data.expand(flags[name], d) + task_deps[name][task] = deptask + getTask('depends') + getTask('deptask') + getTask('rdeptask') + getTask('recrdeptask') + getTask('nostamp') + task_deps['parents'][task] = [] + for dep in flags['deps']: + dep = data.expand(dep, d) + task_deps['parents'][task].append(dep) + + # don't assume holding a reference + data.setVar('_task_deps', task_deps, d) + +def remove_task(task, kill, d): + """Remove an BB 'task'. + + If kill is 1, also remove tasks that depend on this task.""" + + data.delVarFlag(task, 'task', d) + diff --git a/bitbake-dev/lib/bb/cache.py b/bitbake-dev/lib/bb/cache.py new file mode 100644 index 0000000000..bcf393a578 --- /dev/null +++ b/bitbake-dev/lib/bb/cache.py @@ -0,0 +1,465 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake 'Event' implementation +# +# Caching of bitbake variables before task execution + +# Copyright (C) 2006 Richard Purdie + +# but small sections based on code from bin/bitbake: +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os, re +import bb.data +import bb.utils +from sets import Set + +try: + import cPickle as pickle +except ImportError: + import pickle + bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.") + +__cache_version__ = "128" + +class Cache: + """ + BitBake Cache implementation + """ + def __init__(self, cooker): + + + self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True) + self.clean = {} + self.checked = {} + self.depends_cache = {} + self.data = None + self.data_fn = None + self.cacheclean = True + + if self.cachedir in [None, '']: + self.has_cache = False + bb.msg.note(1, bb.msg.domain.Cache, "Not using a cache. Set CACHE = to enable.") + else: + self.has_cache = True + self.cachefile = os.path.join(self.cachedir,"bb_cache.dat") + + bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir) + try: + os.stat( self.cachedir ) + except OSError: + bb.mkdirhier( self.cachedir ) + + if not self.has_cache: + return + + # If any of configuration.data's dependencies are newer than the + # cache there isn't even any point in loading it... + newest_mtime = 0 + deps = bb.data.getVar("__depends", cooker.configuration.data, True) + for f,old_mtime in deps: + if old_mtime > newest_mtime: + newest_mtime = old_mtime + + if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: + try: + p = pickle.Unpickler(file(self.cachefile, "rb")) + self.depends_cache, version_data = p.load() + if version_data['CACHE_VER'] != __cache_version__: + raise ValueError, 'Cache Version Mismatch' + if version_data['BITBAKE_VER'] != bb.__version__: + raise ValueError, 'Bitbake Version Mismatch' + except EOFError: + bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...") + self.depends_cache = {} + except: + bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...") + self.depends_cache = {} + else: + bb.msg.note(1, bb.msg.domain.Cache, "Out of date cache found, rebuilding...") + + def getVar(self, var, fn, exp = 0): + """ + Gets the value of a variable + (similar to getVar in the data class) + + There are two scenarios: + 1. We have cached data - serve from depends_cac