From 27dba1e6247ae48349aee1bce141a9eefaafaad1 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Tue, 9 May 2006 15:44:08 +0000 Subject: Update to bitbake 1.4.2 (latest stable branch release). This includes the caching speedups git-svn-id: https://svn.o-hand.com/repos/poky/trunk@371 311d38ba-8fff-0310-9ca6-ca027cbcb966 --- bitbake/AUTHORS | 3 +- bitbake/ChangeLog | 19 +- bitbake/MANIFEST | 4 + bitbake/bin/bitbake | 272 ++-- bitbake/bin/bitbakec | Bin 38998 -> 0 bytes bitbake/conf/bitbake.conf | 2 +- bitbake/lib/bb/__init__.py | 6 +- bitbake/lib/bb/cache.py | 306 ++++ bitbake/lib/bb/data.py | 273 ++-- bitbake/lib/bb/data_smart.py | 158 +-- bitbake/lib/bb/event.py | 44 +- bitbake/lib/bb/fetch/__init__.py | 4 - bitbake/lib/bb/fetch/git.py | 2 +- bitbake/lib/bb/methodpool.py | 101 ++ bitbake/lib/bb/parse/__init__.py | 6 +- bitbake/lib/bb/parse/parse_c/BBHandler.py | 116 +- bitbake/lib/bb/parse/parse_c/Makefile | 36 + bitbake/lib/bb/parse/parse_c/bitbakec.pyx | 180 +++ bitbake/lib/bb/parse/parse_c/bitbakeparser.cc | 506 ++++--- bitbake/lib/bb/parse/parse_c/bitbakeparser.h | 46 +- bitbake/lib/bb/parse/parse_c/bitbakeparser.y | 14 + bitbake/lib/bb/parse/parse_c/bitbakescanner.cc | 1795 ++++++++++++------------ bitbake/lib/bb/parse/parse_c/bitbakescanner.l | 67 +- bitbake/lib/bb/parse/parse_c/lexer.h | 27 +- bitbake/lib/bb/parse/parse_c/lexerc.h | 17 + bitbake/lib/bb/parse/parse_c/python_output.h | 7 +- bitbake/lib/bb/parse/parse_py/BBHandler.py | 50 +- bitbake/lib/bb/shell.py | 44 +- 28 files changed, 2431 insertions(+), 1674 deletions(-) delete mode 100644 bitbake/bin/bitbakec create mode 100644 bitbake/lib/bb/cache.py create mode 100644 bitbake/lib/bb/methodpool.py create mode 100644 bitbake/lib/bb/parse/parse_c/Makefile create mode 100644 bitbake/lib/bb/parse/parse_c/bitbakec.pyx create mode 100644 bitbake/lib/bb/parse/parse_c/lexerc.h diff --git a/bitbake/AUTHORS b/bitbake/AUTHORS index 4129e4c523..ef119cce22 100644 --- a/bitbake/AUTHORS +++ b/bitbake/AUTHORS @@ -1,5 +1,6 @@ +Phil Blundell Holger Freyther Chris Larson Mickey Lauer +Richard Purdie Holger Schurig -Phil Blundell diff --git a/bitbake/ChangeLog b/bitbake/ChangeLog index 3dfba1ed81..93cc45aaf4 100644 --- a/bitbake/ChangeLog +++ b/bitbake/ChangeLog @@ -1,9 +1,26 @@ -Changes in BitBake 1.3.x: +Changes in BitBake 1.4.2: + - Send logs to oe.pastebin.com instead of pastebin.com + fixes #856 + - Copy the internal bitbake data before building the + dependency graph. This fixes nano not having a + virtual/libc dependency + - Allow multiple TARBALL_STASH entries + - Cache, check if the directory exists before changing + into it + - git speedup cloning by not doing a checkout + - allow to have spaces in filenames (.conf, .bb, .bbclass) + +Changes in BitBake 1.4.0: - Fix to check both RDEPENDS and RDEPENDS_${PN} - Fix a RDEPENDS parsing bug in utils:explode_deps() - Update git fetcher behaviour to match git changes - ASSUME_PROVIDED allowed to include runtime packages - git fetcher cleanup and efficency improvements + - Change the format of the cache + - Update usermanual to document the Fetchers + - Major changes to caching with a new strategy + giving a major performance increase when reparsing + with few data changes Changes in BitBake 1.3.3: - Create a new Fetcher module to ease the diff --git a/bitbake/MANIFEST b/bitbake/MANIFEST index 14a21d7bf4..5823cd0707 100644 --- a/bitbake/MANIFEST +++ b/bitbake/MANIFEST @@ -2,10 +2,12 @@ AUTHORS ChangeLog MANIFEST setup.py +bin/bitdoc bin/bbimage bin/bitbake lib/bb/__init__.py lib/bb/build.py +lib/bb/cache.py lib/bb/data.py lib/bb/data_smart.py lib/bb/event.py @@ -18,6 +20,7 @@ lib/bb/fetch/svk.py lib/bb/fetch/svn.py lib/bb/fetch/wget.py lib/bb/manifest.py +lib/bb/methodpool.py lib/bb/parse/__init__.py lib/bb/parse/parse_py/BBHandler.py lib/bb/parse/parse_py/ConfHandler.py @@ -30,5 +33,6 @@ doc/manual/html.css doc/manual/Makefile doc/manual/usermanual.xml contrib/bbdev.sh +contrib/vim/syntax/bitbake.vim conf/bitbake.conf classes/base.bbclass diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake index 457fbb7527..10cf4bd00a 100755 --- a/bitbake/bin/bitbake +++ b/bitbake/bin/bitbake @@ -24,14 +24,14 @@ import sys, os, getopt, glob, copy, os.path, re, time sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) import bb -from bb import utils, data, parse, debug, event, fatal +from bb import utils, data, parse, debug, event, fatal, cache from sets import Set import itertools, optparse parsespin = itertools.cycle( r'|/-\\' ) bbdebug = 0 -__version__ = "1.3.3.2" +__version__ = "1.4.3" #============================================================================# # BBParsingStatus @@ -44,7 +44,6 @@ class BBParsingStatus: """ def __init__(self): - self.cache_dirty = False self.providers = {} self.rproviders = {} self.packages = {} @@ -60,34 +59,35 @@ class BBParsingStatus: self.pkg_dp = {} self.pn_provides = {} self.all_depends = Set() + self.build_all = {} + self.rundeps = {} + self.runrecs = {} + self.stamp = {} - def handle_bb_data(self, file_name, bb_data, cached): + def handle_bb_data(self, file_name, bb_cache, cached): """ We will fill the dictionaries with the stuff we need for building the tree more fast """ - if bb_data == None: - return - - if not cached: - self.cache_dirty = True - - pn = bb.data.getVar('PN', bb_data, True) - pv = bb.data.getVar('PV', bb_data, True) - pr = bb.data.getVar('PR', bb_data, True) - dp = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0") - provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split()) - depends = (bb.data.getVar("DEPENDS", bb_data, True) or "").split() - packages = (bb.data.getVar('PACKAGES', bb_data, True) or "").split() - packages_dynamic = (bb.data.getVar('PACKAGES_DYNAMIC', bb_data, True) or "").split() - rprovides = (bb.data.getVar("RPROVIDES", bb_data, 1) or "").split() + pn = bb_cache.getVar('PN', file_name, True) + pv = bb_cache.getVar('PV', file_name, True) + pr = bb_cache.getVar('PR', file_name, True) + dp = int(bb_cache.getVar('DEFAULT_PREFERENCE', file_name, True) or "0") + provides = Set([pn] + (bb_cache.getVar("PROVIDES", file_name, True) or "").split()) + depends = (bb_cache.getVar("DEPENDS", file_name, True) or "").split() + packages = (bb_cache.getVar('PACKAGES', file_name, True) or "").split() + packages_dynamic = (bb_cache.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split() + rprovides = (bb_cache.getVar("RPROVIDES", file_name, True) or "").split() # build PackageName to FileName lookup table if pn not in self.pkg_pn: self.pkg_pn[pn] = [] self.pkg_pn[pn].append(file_name) + self.build_all[file_name] = int(bb_cache.getVar('BUILD_ALL_DEPS', file_name, True) or "0") + self.stamp[file_name] = bb_cache.getVar('STAMP', file_name, True) + # build FileName to PackageName lookup table self.pkg_fn[file_name] = pn self.pkg_pvpr[file_name] = (pv,pr) @@ -114,7 +114,7 @@ class BBParsingStatus: if not package in self.packages: self.packages[package] = [] self.packages[package].append(file_name) - rprovides += (bb.data.getVar("RPROVIDES_%s" % package, bb_data, 1) or "").split() + rprovides += (bb_cache.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() for package in packages_dynamic: if not package in self.packages_dynamic: @@ -126,9 +126,32 @@ class BBParsingStatus: self.rproviders[rprovide] = [] self.rproviders[rprovide].append(file_name) + # Build hash of runtime depeneds and rececommends + + def add_dep(deplist, deps): + for dep in deps: + if not dep in deplist: + deplist[dep] = "" + + if not file_name in self.rundeps: + self.rundeps[file_name] = {} + if not file_name in self.runrecs: + self.runrecs[file_name] = {} + + for package in packages + [pn]: + if not package in self.rundeps[file_name]: + self.rundeps[file_name][package] = {} + if not package in self.runrecs[file_name]: + self.runrecs[file_name][package] = {} + + add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RDEPENDS', file_name, True) or "")) + add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RRECOMMENDS', file_name, True) or "")) + add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RDEPENDS_%s" % package, file_name, True) or "")) + add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RRECOMMENDS_%s" % package, file_name, True) or "")) + # Collect files we may need for possible world-dep # calculations - if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True): + if not bb_cache.getVar('BROKEN', file_name, True) and not bb_cache.getVar('EXCLUDE_FROM_WORLD', file_name, True): self.possible_world.append(file_name) @@ -166,7 +189,6 @@ class BBConfiguration( object ): def __init__( self, options ): for key, val in options.__dict__.items(): setattr( self, key, val ) - self.data = data.init() #============================================================================# # BBCooker @@ -190,8 +212,8 @@ class BBCooker: self.stats = BBStatistics() self.status = None - self.pkgdata = None self.cache = None + self.bb_cache = None def tryBuildPackage( self, fn, item, the_data ): """Build one package""" @@ -226,10 +248,11 @@ class BBCooker: If build_depends is empty, we're dealing with a runtime depends """ - the_data = self.pkgdata[fn] + the_data = self.bb_cache.loadDataFull(fn, self) - if not buildAllDeps: - buildAllDeps = bb.data.getVar('BUILD_ALL_DEPS', the_data, True) or False + # Only follow all (runtime) dependencies if doing a build + if not buildAllDeps and self.configuration.cmd is "build": + buildAllDeps = self.status.build_all[fn] # Error on build time dependency loops if build_depends and build_depends.count(fn) > 1: @@ -402,12 +425,15 @@ class BBCooker: print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1], prefstr) + def showEnvironment( self ): """Show the outer or per-package environment""" if self.configuration.buildfile: + self.cb = None + self.bb_cache = bb.cache.init(self) try: - self.configuration.data, fromCache = self.load_bbfile( self.configuration.buildfile ) + self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self) except IOError, e: fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e )) except Exception, e: @@ -457,11 +483,10 @@ class BBCooker: # look to see if one of them is already staged, or marked as preferred. # if so, bump it to the head of the queue for p in providers: - the_data = self.pkgdata[p] - pn = bb.data.getVar('PN', the_data, 1) - pv = bb.data.getVar('PV', the_data, 1) - pr = bb.data.getVar('PR', the_data, 1) - stamp = '%s.do_populate_staging' % bb.data.getVar('STAMP', the_data, 1) + pn = self.status.pkg_fn[p] + pv, pr = self.status.pkg_pvpr[p] + + stamp = '%s.do_populate_staging' % self.status.stamp[p] if os.path.exists(stamp): (newvers, fn) = preferred_versions[pn] if not fn in eligible: @@ -470,11 +495,11 @@ class BBCooker: oldver = "%s-%s" % (pv, pr) newver = '-'.join(newvers) if (newver != oldver): - extra_chat = "; upgrading from %s to %s" % (oldver, newver) + extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item) else: - extra_chat = "" + extra_chat = "Selecting already-staged %s (%s) to satisfy %s" % (pn, oldver, item) if self.configuration.verbose: - bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat)) + bb.note("%s" % extra_chat) eligible.remove(fn) eligible = [fn] + eligible discriminated = True @@ -656,20 +681,11 @@ class BBCooker: rdepends = [] self.rbuild_cache.append(item) - the_data = self.pkgdata[fn] - pn = self.status.pkg_fn[fn] - - if (item == pn): - rdepends += bb.utils.explode_deps(bb.data.getVar('RDEPENDS', the_data, True) or "") - rdepends += bb.utils.explode_deps(bb.data.getVar('RRECOMMENDS', the_data, True) or "") - rdepends += bb.utils.explode_deps(bb.data.getVar("RDEPENDS_%s" % pn, the_data, True) or "") - rdepends += bb.utils.explode_deps(bb.data.getVar('RRECOMMENDS_%s' % pn, the_data, True) or "") - else: - packages = (bb.data.getVar('PACKAGES', the_data, 1).split() or "") - for package in packages: - if package == item: - rdepends += bb.utils.explode_deps(bb.data.getVar("RDEPENDS_%s" % package, the_data, True) or "") - rdepends += bb.utils.explode_deps(bb.data.getVar("RRECOMMENDS_%s" % package, the_data, True) or "") + + if fn in self.status.rundeps and item in self.status.rundeps[fn]: + rdepends += self.status.rundeps[fn][item].keys() + if fn in self.status.runrecs and item in self.status.runrecs[fn]: + rdepends += self.status.runrecs[fn][item].keys() bb.debug(2, "Additional runtime dependencies for %s are: %s" % (item, " ".join(rdepends))) @@ -684,6 +700,9 @@ class BBCooker: all_depends = self.status.all_depends pn_provides = self.status.pn_provides + localdata = data.createCopy(self.configuration.data) + bb.data.update_data(localdata) + def calc_bbfile_priority(filename): for (regex, pri) in self.status.bbfile_config_priorities: if regex.match(filename): @@ -691,17 +710,22 @@ class BBCooker: return 0 # Handle PREFERRED_PROVIDERS - for p in (bb.data.getVar('PREFERRED_PROVIDERS', self.configuration.data, 1) or "").split(): + for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split(): (providee, provider) = p.split(':') if providee in self.preferred and self.preferred[providee] != provider: bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee])) self.preferred[providee] = provider # Calculate priorities for each file - for p in self.pkgdata.keys(): + for p in self.status.pkg_fn.keys(): self.status.bbfile_priority[p] = calc_bbfile_priority(p) - # Build package list for "bitbake world" + def buildWorldTargetList(self): + """ + Build package list for "bitbake world" + """ + all_depends = self.status.all_depends + pn_provides = self.status.pn_provides bb.debug(1, "collating packages for \"world\"") for f in self.status.possible_world: terminal = True @@ -724,9 +748,10 @@ class BBCooker: self.status.possible_world = None self.status.all_depends = None - def myProgressCallback( self, x, y, f, file_data, from_cache ): + def myProgressCallback( self, x, y, f, bb_cache, from_cache ): # feed the status with new input - self.status.handle_bb_data(f, file_data, from_cache) + + self.status.handle_bb_data(f, bb_cache, from_cache) if bbdebug > 0: return @@ -755,6 +780,13 @@ class BBCooker: def parseConfigurationFile( self, afile ): try: self.configuration.data = bb.parse.handle( afile, self.configuration.data ) + + # Add the handlers we inherited by INHERITS + # FIXME: This assumes that we included at least one .inc file + for var in bb.data.keys(self.configuration.data): + if bb.data.getVarFlag(var, 'handler', self.configuration.data): + bb.event.register(var,bb.data.getVar(var,self.configuration.data)) + except IOError: bb.fatal( "Unable to open %s" % afile ) except bb.parse.ParseError, details: @@ -786,6 +818,12 @@ class BBCooker: def cook( self, configuration, args ): + """ + We are building stuff here. We do the building + from here. By default we try to execute task + build. + """ + self.configuration = configuration if not self.configuration.cmd: @@ -801,6 +839,13 @@ class BBCooker: self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) ) + + # + # Special updated configuration we use for firing events + # + self.configuration.event_data = bb.data.createCopy(self.configuration.data) + bb.data.update_data(self.configuration.event_data) + if self.configuration.show_environment: self.showEnvironment() sys.exit( 0 ) @@ -876,18 +921,18 @@ class BBCooker: print "Requested parsing .bb files only. Exiting." return - bb.data.update_data( self.configuration.data ) self.buildDepgraph() if self.configuration.show_versions: self.showVersions() sys.exit( 0 ) if 'world' in pkgs_to_build: + self.buildWorldTargetList() pkgs_to_build.remove('world') for t in self.status.world_target: pkgs_to_build.append(t) - bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.data)) + bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data)) failures = 0 for k in pkgs_to_build: @@ -905,7 +950,7 @@ class BBCooker: if self.configuration.abort: sys.exit(1) - bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.data, failures)) + bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures)) sys.exit( self.stats.show() ) @@ -932,77 +977,12 @@ class BBCooker: return [] return finddata.readlines() - def deps_clean(self, d): - depstr = data.getVar('__depends', d) - if depstr: - deps = depstr.split(" ") - for dep in deps: - (f,old_mtime_s) = dep.split("@") - old_mtime = int(old_mtime_s) - new_mtime = parse.cached_mtime(f) - if (new_mtime > old_mtime): - return False - return True - - def load_bbfile( self, bbfile ): - """Load and parse one .bb build file""" - - if not self.cache in [None, '']: - # get the times - cache_mtime = data.init_db_mtime(self.cache, bbfile) - file_mtime = parse.cached_mtime(bbfile) - - if file_mtime > cache_mtime: - #print " : '%s' dirty. reparsing..." % bbfile - pass - else: - #print " : '%s' clean. loading from cache..." % bbfile - cache_data = data.init_db( self.cache, bbfile, False ) - if self.deps_clean(cache_data): - return cache_data, True - - topdir = data.getVar('TOPDIR', self.configuration.data) - if not topdir: - topdir = os.path.abspath(os.getcwd()) - # set topdir to here - data.setVar('TOPDIR', topdir, self.configuration) - bbfile = os.path.abspath(bbfile) - bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) - # expand tmpdir to include this topdir - data.setVar('TMPDIR', data.getVar('TMPDIR', self.configuration.data, 1) or "", self.configuration.data) - # set topdir to location of .bb file - topdir = bbfile_loc - #data.setVar('TOPDIR', topdir, cfg) - # go there - oldpath = os.path.abspath(os.getcwd()) - os.chdir(topdir) - bb = data.init_db(self.cache,bbfile, True, self.configuration.data) - try: - parse.handle(bbfile, bb) # read .bb data - if not self.cache in [None, '']: - bb.commit(parse.cached_mtime(bbfile)) # write cache - os.chdir(oldpath) - return bb, False - finally: - os.chdir(oldpath) - def collect_bbfiles( self, progressCallback ): """Collect all available .bb build files""" self.cb = progressCallback parsed, cached, skipped, masked = 0, 0, 0, 0 - self.cache = bb.data.getVar( "CACHE", self.configuration.data, 1 ) - self.pkgdata = data.pkgdata( not self.cache in [None, ''], self.cache, self.configuration.data ) + self.bb_cache = bb.cache.init(self) - if not self.cache in [None, '']: - if self.cb is not None: - print "NOTE: Using cache in '%s'" % self.cache - try: - os.stat( self.cache ) - except OSError: - bb.mkdirhier( self.cache ) - else: - if self.cb is not None: - print "NOTE: Not using a cache. Set CACHE = to enable." files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split() data.setVar("BBFILES", " ".join(files), self.configuration.data) @@ -1037,43 +1017,49 @@ class BBCooker: # read a file's metadata try: - bb_data, fromCache = self.load_bbfile(f) - if fromCache: cached += 1 + fromCache, skip = self.bb_cache.loadData(f, self) + if skip: + skipped += 1 + #bb.note("Skipping %s" % f) + self.bb_cache.skip(f) + continue + elif fromCache: cached += 1 else: parsed += 1 deps = None - if bb_data is not None: - # allow metadata files to add items to BBFILES - #data.update_data(self.pkgdata[f]) - addbbfiles = data.getVar('BBFILES', bb_data) or None - if addbbfiles: - for aof in addbbfiles.split(): - if not files.count(aof): - if not os.path.isabs(aof): - aof = os.path.join(os.path.dirname(f),aof) - files.append(aof) - for var in bb_data.keys(): - if data.getVarFlag(var, "handler", bb_data) and data.getVar(var, bb_data): - event.register(data.getVar(var, bb_data)) - self.pkgdata[f] = bb_data + + # allow metadata files to add items to BBFILES + #data.update_data(self.pkgdata[f]) + addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None + if addbbfiles: + for aof in addbbfiles.split(): + if not files.count(aof): + if not os.path.isabs(aof): + aof = os.path.join(os.path.dirname(f),aof) + files.append(aof) # now inform the caller if self.cb is not None: - self.cb( i + 1, len( newfiles ), f, bb_data, fromCache ) + self.cb( i + 1, len( newfiles ), f, self.bb_cache, fromCache ) except IOError, e: + self.bb_cache.remove(f) bb.error("opening %s: %s" % (f, e)) pass - except bb.parse.SkipPackage: - skipped += 1 - pass except KeyboardInterrupt: + self.bb_cache.sync() raise except Exception, e: + self.bb_cache.remove(f) bb.error("%s while parsing %s" % (e, f)) + except: + self.bb_cache.remove(f) + raise if self.cb is not None: print "\rNOTE: Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ), + self.bb_cache.sync() + #============================================================================# # main #============================================================================# diff --git a/bitbake/bin/bitbakec b/bitbake/bin/bitbakec deleted file mode 100644 index db6a160303..0000000000 Binary files a/bitbake/bin/bitbakec and /dev/null differ diff --git a/bitbake/conf/bitbake.conf b/bitbake/conf/bitbake.conf index fd216a3261..d288fee78f 100644 --- a/bitbake/conf/bitbake.conf +++ b/bitbake/conf/bitbake.conf @@ -51,5 +51,5 @@ T = "${WORKDIR}/temp" TARGET_ARCH = "${BUILD_ARCH}" TMPDIR = "${TOPDIR}/tmp" UPDATECOMMAND = "" -UPDATECOMMAND_cvs = "/usr/bin/env cvs update ${CVSCOOPTS}" +UPDATECOMMAND_cvs = "/usr/bin/env cvs -d${CVSROOT} update ${CVSCOOPTS}" WORKDIR = "${TMPDIR}/work/${PF}" diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py index c6c0beb792..c3e7a16658 100644 --- a/bitbake/lib/bb/__init__.py +++ b/bitbake/lib/bb/__init__.py @@ -23,7 +23,7 @@ this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. """ -__version__ = "1.3.3.4" +__version__ = "1.4.3" __all__ = [ @@ -60,7 +60,9 @@ __all__ = [ "event", "build", "fetch", - "manifest" + "manifest", + "methodpool", + "cache", ] whitespace = '\t\n\x0b\x0c\r ' diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py new file mode 100644 index 0000000000..921a9f7589 --- /dev/null +++ b/bitbake/lib/bb/cache.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Event' implementation + +Caching of bitbake variables before task execution + +# Copyright (C) 2006 Richard Purdie + +# but small sections based on code from bin/bitbake: +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +""" + +import os, re +import bb.data +import bb.utils + +try: + import cPickle as pickle +except ImportError: + import pickle + print "NOTE: Importing cPickle failed. Falling back to a very slow implementation." + +# __cache_version__ = "123" +__cache_version__ = "124" # changes the __depends structure + +class Cache: + """ + BitBake Cache implementation + """ + def __init__(self, cooker): + + + self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True) + self.clean = {} + self.depends_cache = {} + self.data = None + self.data_fn = None + + if self.cachedir in [None, '']: + self.has_cache = False + if cooker.cb is not None: + print "NOTE: Not using a cache. Set CACHE = to enable." + else: + self.has_cache = True + self.cachefile = os.path.join(self.cachedir,"bb_cache.dat") + + if cooker.cb is not None: + print "NOTE: Using cache in '%s'" % self.cachedir + try: + os.stat( self.cachedir ) + except OSError: + bb.mkdirhier( self.cachedir ) + + if self.has_cache and (self.mtime(self.cachefile)): + try: + p = pickle.Unpickler( file(self.cachefile,"rb")) + self.depends_cache, version_data = p.load() + if version_data['CACHE_VER'] != __cache_version__: + raise ValueError, 'Cache Version Mismatch' + if version_data['BITBAKE_VER'] != bb.__version__: + raise ValueError, 'Bitbake Version Mismatch' + except (ValueError, KeyError): + bb.note("Invalid cache found, rebuilding...") + self.depends_cache = {} + + if self.depends_cache: + for fn in self.depends_cache.keys(): + self.clean[fn] = "" + self.cacheValidUpdate(fn) + + def getVar(self, var, fn, exp = 0): + """ + Gets the value of a variable + (similar to getVar in the data class) + + There are two scenarios: + 1. We have cached data - serve from depends_cache[fn] + 2. We're learning what data to cache - serve from data + backend but add a copy of the data to the cache. + """ + + if fn in self.clean: + return self.depends_cache[fn][var] + + if not fn in self.depends_cache: + self.depends_cache[fn] = {} + + if fn != self.data_fn: + # We're trying to access data in the cache which doesn't exist + # yet setData hasn't been called to setup the right access. Very bad. + bb.error("Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn)) + + result = bb.data.getVar(var, self.data, exp) + self.depends_cache[fn][var] = result + return result + + def setData(self, fn, data): + """ + Called to prime bb_cache ready to learn which variables to cache. + Will be followed by calls to self.getVar which aren't cached + but can be fulfilled from self.data. + """ + self.data_fn = fn + self.data = data + + # Make sure __depends makes the depends_cache + self.getVar("__depends", fn, True) + self.depends_cache[fn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn) + + def loadDataFull(self, fn, cooker): + """ + Return a complete set of data for fn. + To do this, we need to parse the file. + """ + bb_data, skipped = self.load_bbfile(fn, cooker) + return bb_data + + def loadData(self, fn, cooker): + """ + Load a subset of data for fn. + If the cached data is valid we do nothing, + To do this, we need to parse the file and set the system + to record the variables accessed. + Return the cache status and whether the file was skipped when parsed + """ + if self.cacheValid(fn): + if "SKIPPED" in self.depends_cache[fn]: + return True, True + return True, False + + bb_data, skipped = self.load_bbfile(fn, cooker) + self.setData(fn, bb_data) + return False, skipped + + def cacheValid(self, fn): + """ + Is the cache valid for fn? + Fast version, no timestamps checked. + """ + # Is cache enabled? + if not self.has_cache: + return False + if fn in self.clean: + return True + return False + + def cacheValidUpdate(self, fn): + """ + Is the cache valid for fn? + Make thorough (slower) checks including timestamps. + """ + # Is cache enabled? + if not self.has_cache: + return False + + # Check file still exists + if self.mtime(fn) == 0: + bb.debug(2, "Cache: %s not longer exists" % fn) + self.remove(fn) + return False + + # File isn't in depends_cache + if not fn in self.depends_cache: + bb.debug(2, "Cache: %s is not cached" % fn) + self.remove(fn) + return False + + # Check the file's timestamp + if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True): + bb.debug(2, "Cache: %s changed" % fn) + self.remove(fn) + return False + + # Check dependencies are still valid + depends = self.getVar("__depends", fn, True) + for f,old_mtime in depends: + new_mtime = bb.parse.cached_mtime(f) + if (new_mtime > old_mtime): + bb.debug(2, "Cache: %s's dependency %s changed" % (fn, f)) + self.remove(fn) + return False + + bb.debug(2, "Depends Cache: %s is clean" % fn) + if not fn in self.clean: + self.clean[fn] = "" + + return True + + def skip(self, fn): + """ + Mark a fn as skipped + Called from the parser + """ + if not fn in self.depends_cache: + self.depends_cache[fn] = {} + self.depends_cache[fn]["SKIPPED"] = "1" + + def remove(self, fn): + """ + Remove a fn from the cache + Called from the parser in error cases + """ + bb.debug(1, "Removing %s from cache" % fn) + if fn in self.depends_cache: + del self.depends_cache[fn] + if fn in self.clean: + del self.clean[fn] + + def sync(self): + """ + Save the cache + Called from the parser when complete (or exitting) + """ + + if not self.has_cache: + return + + version_data = {} + version_data['CACHE_VER'] = __cache_version__ + version_data['BITBAKE_VER'] = bb.__version__ + + p = pickle.Pickler(file(self.cachefile, "wb" ), -1 ) + p.dump([self.depends_cache, version_data]) + + def mtime(self, cachefile): + try: + return os.stat(cachefile)[8] + except OSError: + return 0 + + def load_bbfile( self, bbfile , cooker): + """ + Load and parse one .bb build file + Return the data and whether parsing resulted in the file being skipped + """ + + import bb + from bb import utils, data, parse, debug, event, fatal + + topdir = data.getVar('TOPDIR', cooker.configuration.data) + if not topdir: + topdir = os.path.abspath(os.getcwd()) + # set topdir to here + data.setVar('TOPDIR', topdir, cooker.configuration) + bbfile = os.path.abspath(bbfile) + bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) + # expand tmpdir to include this topdir + data.setVar('TMPDIR', data.getVar('TMPDIR', cooker.configuration.data, 1) or "", cooker.configuration.data) + # set topdir to location of .bb file + topdir = bbfile_loc + #data.setVar('TOPDIR', topdir, cfg) + # go there + oldpath = os.path.abspath(os.getcwd()) + if self.mtime(topdir): + os.chdir(topdir) + bb_data = data.init_db(cooker.configuration.data) + try: + parse.handle(bbfile, bb_data) # read .bb data + os.chdir(oldpath) + return bb_data, False + except bb.parse.SkipPackage: + os.chdir(oldpath) + return bb_data, True + except: + os.chdir(oldpath) + raise + +def init(cooker): + """ + The Objective: Cache the minimum amount of data possible yet get to the + stage of building packages (i.e. tryBuild) without reparsing any .bb files. + + To do this, we intercept getVar calls and only cache the variables we see + being accessed. We rely on the cache getVar calls being made for all + variables bitbake might need to use to reach this stage. For each cached + file we need to track: + + * Its mtime + * The mtimes of all its dependencies + * Whether it caused a parse.SkipPackage exception + + Files causing parsing errors are evicted from the cache. + + """ + return Cache(cooker) + diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py index 56ee977f66..55d1cc9053 100644 --- a/bitbake/lib/bb/data.py +++ b/bitbake/lib/bb/data.py @@ -7,6 +7,18 @@ BitBake 'Data' implementations Functions for interacting with the data structure used by the BitBake build tools. +The expandData and update_data are the most expensive +operations. At night the cookie monster came by and +suggested 'give me cookies on setting the variables and +things will work out'. Taking this suggestion into account +applying the skills from the not yet passed 'Entwurf und +Analyse von Algorithmen' lecture and the cookie +monster seems to be right. We will track setVar more carefully +to have faster update_data and expandKeys operations. + +This is a treade-off between speed and memory again but +the speed is more critical here. + Copyright (C) 2003, 2004 Chris Larson Copyright (C) 2005 Holger Hans Peter Freyther @@ -36,88 +48,15 @@ sys.path.insert(0,path) from bb import note, debug, data_smart _dict_type = data_smart.DataSmart -_dict_p_type = data_smart.DataSmartPackage - -class DataDictFull(dict): - """ - This implements our Package Data Storage Interface. - setDirty is a no op as all items are held in memory - """ - def setDirty(self, bbfile, data): - """ - No-Op we assume data was manipulated as some sort of - reference - """ - if not bbfile in self: - raise Exception("File %s was not in dictionary before" % bbfile) - - self[bbfile] = data - -class DataDictCache: - """ - Databacked Dictionary implementation - """ - def __init__(self, cache_dir, config): - self.cache_dir = cache_dir - self.files = [] - self.dirty = {} - self.config = config - - def has_key(self,key): - return key in self.files - - def keys(self): - return self.files - - def __setitem__(self, key, data): - """ - Add the key to the list of known files and - place the data in the cache? - """ - if key in self.files: - return - - self.files.append(key) - - def __getitem__(self, key): - if not key in self.files: - return None - - # if it was dirty we will - if key in self.dirty: - return self.dirty[key] - - # not cached yet - return _dict_p_type(self.cache_dir, key,False,self.config) - - def setDirty(self, bbfile, data): - """ - Only already added items can be declared dirty!!! - """ - - if not bbfile in self.files: - raise Exception("File %s was not in dictionary before" % bbfile) - - self.dirty[bbfile] = data - - def init(): return _dict_type() -def init_db(cache,name,clean,parent = None): - return _dict_p_type(cache,name,clean,parent) - -def init_db_mtime(cache,cache_bbfile): - return _dict_p_type.mtime(cache,cache_bbfile) - -def pkgdata(use_cache, cache, config = None): - """ - Return some sort of dictionary to lookup parsed dictionaires - """ - if use_cache: - return DataDictCache(cache, config) - return DataDictFull() +def init_db(parent = None): + if parent: + return parent.createCopy() + else: + return _dict_type() def createCopy(source): """Link the source set to the destination @@ -273,6 +212,27 @@ def setData(newData, d): """Sets the data object to the supplied value""" d = newData + +## +## Cookie Monsters' query functions +## +def _get_override_vars(d, override): + """ + Internal!!! + + Get the Names of Variables that have a specific + override. This function returns a iterable + Set or an empty list + """ + return [] + +def _get_var_flags_triple(d): + """ + Internal!!! + + """ + return [] + __expand_var_regexp__ = re.compile(r"\${[^{}]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") @@ -303,43 +263,7 @@ def expand(s, d, varname = None): >>> print expand('${SRC_URI}', d) http://somebug.${TARGET_MOO} """ - def var_sub(match): - key = match.group()[2:-1] - if varname and key: - if varname == key: - raise Exception("variable %s references itself!" % varname) - var = getVar(key, d, 1) - if var is not None: - return var - else: - return match.group() - - def python_sub(match): - import bb - code = match.group()[3:-1] - locals()['d'] = d - s = eval(code) - if type(s) == types.IntType: s = str(s) - return s - - if type(s) is not types.StringType: # sanity check - return s - - while s.find('$') != -1: - olds = s - try: - s = __expand_var_regexp__.sub(var_sub, s) - s = __expand_python_regexp__.sub(python_sub, s) - if s == olds: break - if type(s) is not types.StringType: # sanity check - import bb - bb.error('expansion of %s returned non-string %s' % (olds, s)) - except KeyboardInterrupt: - raise - except: - note("%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s)) - raise - return s + return d.expand(s, varname) def expandKeys(alterdata, readdata = None): if readdata == None: @@ -356,7 +280,7 @@ def expandKeys(alterdata, readdata = None): # setVarFlags(ekey, copy.copy(getVarFlags(key, readdata)), alterdata) setVar(ekey, val, alterdata) - for i in ('_append', '_prepend', '_delete'): + for i in ('_append', '_prepend'): dest = getVarFlag(ekey, i, alterdata) or [] src = getVarFlag(key, i, readdata) or [] dest.extend(src) @@ -507,67 +431,76 @@ def update_data(d): >>> print getVar('TEST', d) local """ - debug(2, "update_data()") -# can't do delete env[...] while iterating over the dictionary, so remember them - dodel = [] + # now ask the cookie monster for help + #print "Cookie Monster" + #print "Append/Prepend %s" % d._special_values + #print "Overrides %s" % d._seen_overrides + overrides = (getVar('OVERRIDES', d, 1) or "").split(':') or [] - def applyOverrides(var, d): - if not overrides: - debug(1, "OVERRIDES not defined, nothing to do") - return - val = getVar(var, d) - for o in overrides: - if var.endswith("_" + o): - l = len(o)+1 - name = var[:-l] - d[name] = d[var] + # + # Well let us see what breaks here. We used to iterate + # over each variable and apply the override and then + # do the line expanding. + # If we have bad luck - which we will have - the keys + # where in some order that is so important for this + # method which we don't have anymore. + # Anyway we will fix that and write test cases this + # time. + + # + # First we apply all overrides + # Then we will handle _append and _prepend + # + + for o in overrides: + # calculate '_'+override + l = len(o)+1 + + # see if one should even try + if not o in d._seen_overrides: + continue - for s in keys(d): - applyOverrides(s, d) - sval = getVar(s, d) or "" - -# Handle line appends: - for (a, o) in getVarFlag(s, '_append', d) or []: - # maybe the OVERRIDE was not yet added so keep the append - if (o and o in overrides) or not o: - delVarFlag(s, '_append', d) - if o: - if not o in overrides: + vars = d._seen_overrides[o] + for var in vars: + name = var[:-l] + try: + d[name] = d[var] + except: + note ("Untracked delVar") + + # now on to the appends and prepends + if '_append' in d._special_values: + appends = d._special_values['_append'] or [] + for append in appends: + for (a, o) in getVarFlag(append, '_append', d) or []: + # maybe the OVERRIDE was not yet added so keep the append + if (o and o in overrides) or not o: + delVarFlag(append, '_append', d) + if o and not o in overrides: continue - sval+=a - setVar(s, sval, d) - -# Handle line prepends - for (a, o) in getVarFlag(s, '_prepend', d) or []: - # maybe the OVERRIDE was not yet added so keep the append - if (o and o in overrides) or not o: - delVarFlag(s, '_prepend', d) - if o: - if not o in overrides: + + sval = getVar(append,d) or "" + sval+=a + setVar(append, sval, d) + + + if '_prepend' in d._special_values: + prepends = d._special_values['_prepend'] or [] + + for prepend in prepends: + for (a, o) in getVarFlag(prepend, '_prepend', d) or []: + # maybe the OVERRIDE was not yet added so keep the prepend + if (o and o in overrides) or not o: + delVarFlag(prepend, '_prepend', d) + if o and not o in overrides: continue - sval=a+sval - setVar(s, sval, d) - -# Handle line deletions - name = s + "_delete" - nameval = getVar(name, d) - if nameval: - sval = getVar(s, d) - if sval: - new = '' - pattern = nameval.replace('\n','').strip() - for line in sval.split('\n'): - if line.find(pattern) == -1: - new = new + '\n' + line - setVar(s, new, d) - dodel.append(name) - -# delete all environment vars no longer needed - for s in dodel: - delVar(s, d) + + sval = a + (getVar(prepend,d) or "") + setVar(prepend, sval, d) + def inherits_class(klass, d): val = getVar('__inherit_cache', d) or "" diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index 52f391dec1..fbd4167fe4 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py @@ -8,7 +8,7 @@ BitBake build tools. Copyright (C) 2003, 2004 Chris Larson Copyright (C) 2004, 2005 Seb Frankengul -Copyright (C) 2005 Holger Hans Peter Freyther +Copyright (C) 2005, 2006 Holger Hans Peter Freyther Copyright (C) 2005 Uli Luckas Copyright (C) 2005 ROAD GmbH @@ -29,7 +29,8 @@ Based on functions from the base bb module, Copyright 2003 Holger Schurig """ import copy, os, re, sys, time, types -from bb import note, debug, fatal, utils +from bb import note, debug, error, fatal, utils, methodpool +from sets import Set try: import cPickle as pickle @@ -37,9 +38,8 @@ except ImportError: import pickle print "NOTE: Importing cPickle failed. Falling back to a very slow implementation." - -__setvar_keyword__ = ["_append","_prepend","_delete"] -__setvar_regexp__ = re.compile('(?P.*?)(?P_append|_prepend|_delete)(_(?P.*))?') +__setvar_keyword__ = ["_append","_prepend"] +__setvar_regexp__ = re.compile('(?P.*?)(?P_append|_prepend)(_(?P.*))?') __expand_var_regexp__ = re.compile(r"\${[^{}]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") @@ -48,6 +48,10 @@ class DataSmart: def __init__(self): self.dict = {} + # cookie monster tribute + self._special_values = {} + self._seen_overrides = {} + def expand(self,s, varname): def var_sub(match): key = match.group()[2:-1] @@ -78,8 +82,7 @@ class DataSmart: s = __expand_python_regexp__.sub(python_sub, s) if s == olds: break if type(s) is not types.StringType: # sanity check - import bb - bb.error('expansion of %s returned non-string %s' % (olds, s)) + error('expansion of %s returned non-string %s' % (olds, s)) except KeyboardInterrupt: raise except: @@ -91,18 +94,6 @@ class DataSmart: if not var in self.dict: self.dict[var] = {} - def pickle_prep(self, cfg): - if "_data" in self.dict: - if self.dict["_data"] == cfg: - self.dict["_data"] = "cfg"; - else: # this is an unknown array for the moment - pass - - def unpickle_prep(self, cfg): - if "_data" in self.dict: - if self.dict["_data"] == "cfg": - self.dict["_data"] = cfg; - def _findVar(self,var): _dest = self.dict @@ -116,14 +107,6 @@ class DataSmart: return _dest[var] return None - def _copyVar(self,var,name): - local_var = self._findVar(var) - if local_var: - self.dict[name] = copy.copy(local_var) - else: - debug(1,"Warning, _copyVar %s to %s, %s does not exists" % (var,name,var)) - - def _makeShadowCopy(self, var): if var in self.dict: return @@ -142,11 +125,20 @@ class DataSmart: keyword = match.group("keyword") override = match.group('add') l = self.getVarFlag(base, keyword) or [] - if override == 'delete': - if l.count([value, None]): - del l[l.index([value, None])] l.append([value, override]) - self.setVarFlag(base, match.group("keyword"), l) + self.setVarFlag(base, keyword, l) + + # pay the cookie monster + try: + self._special_values[keyword].add( base ) + except: + self._special_values[keyword] = Set() + self._special_values[keyword].add( base ) + + # SRC_URI_append_simpad is both a flag and a override + #if not override in self._seen_overrides: + # self._seen_overrides[override] = Set() + #self._seen_overrides[override].add( base ) return if not var in self.dict: @@ -155,6 +147,13 @@ class DataSmart: self.delVarFlag(var, 'matchesenv') self.setVarFlag(var, 'export', 1) + # more cookies for the cookie monster + if '_' in var: + override = var[var.rfind('_')+1:] + if not override in self._seen_overrides: + self._seen_overrides[override] = Set() + self._seen_overrides[override].add( var ) + # setting var self.dict[var]["content"] = value @@ -237,6 +236,8 @@ class DataSmart: # we really want this to be a DataSmart... data = DataSmart() data.dict["_data"] = self.dict + data._seen_overrides = copy.deepcopy(self._seen_overrides) + data._special_values = copy.deepcopy(self._special_values) return data @@ -254,98 +255,11 @@ class DataSmart: return keytab.keys() def __getitem__(self,item): - start = self.dict - while start: - if item in start: - return start[item] - elif "_data" in start: - start = start["_data"] - else: - start = None - return None + #print "Warning deprecated" + return self.getVar(item, False) def __setitem__(self,var,data): - self._makeShadowCopy(var) - self.dict[var] = data - - -class DataSmartPackage(DataSmart): - """ - Persistent Data Storage - """ - def sanitize_filename(bbfile): - return bbfile.replace( '/', '_' ) - sanitize_filename = staticmethod(sanitize_filename) + #print "Warning deprecated" + self.setVar(var,data) - def unpickle(self): - """ - Restore the dict from memory - """ - cache_bbfile = self.sanitize_filename(self.bbfile) - p = pickle.Unpickler( file("%s/%s"%(self.cache,cache_bbfile),"rb")) - self.dict = p.load() - self.unpickle_prep() - funcstr = self.getVar('__functions__', 0) - if funcstr: - comp = utils.better_compile(funcstr, "", self.bbfile) - utils.better_exec(comp, __builtins__, funcstr, self.bbfile) - - def linkDataSet(self): - if not self.parent == None: - # assume parent is a DataSmartInstance - self.dict["_data"] = self.parent.dict - - - def __init__(self,cache,name,clean,parent): - """ - Construct a persistent data instance - """ - #Initialize the dictionary - DataSmart.__init__(self) - - self.cache = cache - self.bbfile = os.path.abspath( name ) - self.parent = parent - - # Either unpickle the data or do copy on write - if clean: - self.linkDataSet() - else: - self.unpickle() - - def commit(self, mtime): - """ - Save the package to a permanent storage - """ - self.pickle_prep() - - cache_bbfile = self.sanitize_filename(self.bbfile) - p = pickle.Pickler(file("%s/%s" %(self.cache,cache_bbfile), "wb" ), -1 ) - p.dump( self.dict ) - - self.unpickle_prep() - def mtime(cache,bbfile): - cache_bbfile = DataSmartPackage.sanitize_filename(bbfile) - try: - return os.stat( "%s/%s" % (cache,cache_bbfile) )[8] - except OSError: - return 0 - mtime = staticmethod(mtime) - - def pickle_prep(self): - """ - If self.dict contains a _data key and it is a configuration - we will remember we had a configuration instance attached - """ - if "_data" in self.dict: - if self.dict["_data"] == self.parent: - dest["_data"] = "cfg" - - def unpickle_prep(self): - """ - If we had a configuration instance attached, we will reattach it - """ - if "_data" in self.dict: - if self.dict["_data"] == "cfg": - self.dict["_data"] = self.parent diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py index cbe6d2a11a..b1d12177c4 100644 --- a/bitbake/lib/bb/event.py +++ b/bitbake/lib/bb/event.py @@ -44,7 +44,13 @@ class Event: NotHandled = 0 Handled = 1 -handlers = [] + +Registered = 10 +AlreadyRegistered = 14 + +# Internal +_handlers = [] +_handlers_dict = {} def tmpHandler(event): """Default handler for code events""" @@ -57,7 +63,7 @@ def defaultTmpHandler(): def fire(event): """Fire off an Event""" - for h in handlers: + for h in _handlers: if type(h).__name__ == "code": exec(h) if tmpHandler(event) == Handled: @@ -67,15 +73,22 @@ def fire(event): return Handled return NotHandled -def register(handler): +def register(name, handler): """Register an Event handler""" + + # already registered + if name in _handlers_dict: + return AlreadyRegistered + if handler is not None: # handle string containing python code if type(handler).__name__ == "str": - return _registerCode(handler) -# prevent duplicate registration - if not handler in handlers: - handlers.append(handler) + _registerCode(handler) + else: + _handlers.append(handler) + + _handlers_dict[name] = 1 + return Registered def _registerCode(handlerStr): """Register a 'code' Event. @@ -88,24 +101,23 @@ def _registerCode(handlerStr): tmp = "def tmpHandler(e):\n%s" % handlerStr comp = bb.utils.better_compile(tmp, "tmpHandler(e)", "bb.event._registerCode") # prevent duplicate registration - if not comp in handlers: - handlers.append(comp) + _handlers.append(comp) -def remove(handler): +def remove(name, handler): """Remove an Event handler""" - for h in handlers: - if type(handler).__name__ == "str": - return _removeCode(handler) - if handler is h: - handlers.remove(handler) + _handlers_dict.pop(name) + if type(handler).__name__ == "str": + return _removeCode(handler) + else: + _handlers.remove(handler) def _removeCode(handlerStr): """Remove a 'code' Event handler Deprecated interface; call remove instead.""" tmp = "def tmpHandler(e):\n%s" % handlerStr comp = bb.utils.better_compile(tmp, "tmpHandler(e)", "bb.event._removeCode") - handlers.remove(comp) + _handlers.remove(comp) def getName(e): """Returns the name of a class or class instance""" diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py index 0515f2a5e9..ac698a0d1c 100644 --- a/bitbake/lib/bb/fetch/__init__.py +++ b/bitbake/lib/bb/fetch/__init__.py @@ -168,10 +168,6 @@ class Fetch(object): d Is a bb.data instance tarfn is the name of the tarball """ - tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn) - if os.access(tarpath, os.R_OK): - return True - pn = data.getVar('PN', d, True) src_tarball_stash = None if pn: diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py index f30ae2360a..49235c141e 100644 --- a/bitbake/lib/bb/fetch/git.py +++ b/bitbake/lib/bb/fetch/git.py @@ -129,7 +129,7 @@ class Git(Fetch): os.chdir(repodir) rungitcmd("tar -xzf %s" % (repofile),d) else: - rungitcmd("git clone %s://%s%s %s" % (proto, host, path, repodir),d) + rungitcmd("git clone -n %s://%s%s %s" % (proto, host, path, repodir),d) os.chdir(repodir) rungitcmd("git pull %s://%s%s" % (proto, host, path),d) diff --git a/bitbake/lib/bb/methodpool.py b/bitbake/lib/bb/methodpool.py new file mode 100644 index 0000000000..d7434ed33e --- /dev/null +++ b/bitbake/lib/bb/methodpool.py @@ -0,0 +1,101 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# +# Copyright (C) 2006 Holger Hans Peter Freyther +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# Neither the name Holger Hans Peter Freyther nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + + +""" + What is a method pool? + + BitBake has a global method scope where .bb, .inc and .bbclass + files can install methods. These methods are parsed from strings. + To avoid recompiling and executing these string we introduce + a method pool to do this task. + + This pool will be used to compile and execute the functions. It + will be smart enough to +""" + +from bb.utils import better_compile, better_exec +from bb import error + +# A dict of modules we have handled +# it is the number of .bbclasses + x in size +_parsed_methods = { } +_parsed_fns = { } + +def insert_method(modulename, code, fn): + """ + Add code of a module should be added. The methods + will be simply added, no checking will be done + """ + comp = better_compile(code, "", fn ) + better_exec(comp, __builtins__, code, fn) + + # hack hack hack XXX + return + + # now some instrumentation + code = comp.co_names + for name in code: + if name in ['None', 'False']: + continue + elif name in _parsed_fns and not _parsed_fns[name] == modulename: + error( "Error Method already seen: %s in' %s' now in '%s'" % (name, _parsed_fns[name], modulename)) + else: + _parsed_fns[name] = modulename + +def check_insert_method(modulename, code, fn): + """ + Add the code if it wasnt added before. The module + name will be used for that + + Variables: + @modulename a short name e.g. base.bbclass + @code The actual python code + @fn The filename from the outer file + """ + if not modulename in _parsed_methods: + return insert_method(modulename, code, fn) + +def parsed_module(modulename): + """ + Inform me file xyz was parsed + """ + return modulename in _parsed_methods + + +def get_parsed_dict(): + """ + shortcut + """ + return _parsed_methods diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py index cb27416061..58e17d154a 100644 --- a/bitbake/lib/bb/parse/__init__.py +++ b/bitbake/lib/bb/parse/__init__.py @@ -46,9 +46,9 @@ def update_mtime(f): def mark_dependency(d, f): if f.startswith('./'): f = "%s/%s" % (os.getcwd(), f[2:]) - deps = (bb.data.getVar('__depends', d) or "").split() - deps.append("%s@%s" % (f, cached_mtime(f))) - bb.data.setVar('__depends', " ".join(deps), d) + deps = bb.data.getVar('__depends', d) or [] + deps.append( (f, cached_mtime(f)) ) + bb.data.setVar('__depends', deps, d) def supports(fn, data): """Returns true if we have a handler for this file, false otherwise""" diff --git a/bitbake/lib/bb/parse/parse_c/BBHandler.py b/bitbake/lib/bb/parse/parse_c/BBHandler.py index 300871d9e3..d9f48db17b 100644 --- a/bitbake/lib/bb/parse/parse_c/BBHandler.py +++ b/bitbake/lib/bb/parse/parse_c/BBHandler.py @@ -1,29 +1,42 @@ # ex:ts=4:sw=4:sts=4:et # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# Copyright (C) 2006 Holger Hans Peter Freyther -# -# Permission is hereb