diff options
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
| -rw-r--r-- | bitbake/lib/bb/cooker.py | 436 |
1 files changed, 232 insertions, 204 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 95f38f6236..e524db7498 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # ex:ts=4:sw=4:sts=4:et # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- # @@ -23,37 +24,36 @@ from __future__ import print_function import sys, os, glob, os.path, re, time +import atexit +import itertools +import logging +import multiprocessing +import signal import sre_constants +import threading from cStringIO import StringIO from contextlib import closing import bb from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue +logger = logging.getLogger("BitBake") +collectlog = logging.getLogger("BitBake.Collection") +buildlog = logging.getLogger("BitBake.Build") +parselog = logging.getLogger("BitBake.Parsing") +providerlog = logging.getLogger("BitBake.Provider") + class MultipleMatches(Exception): """ Exception raised when multiple file matches are found """ -class ParsingErrorsFound(Exception): - """ - Exception raised when parsing errors are found - """ - class NothingToBuild(Exception): """ Exception raised when there is nothing to build """ - -# Different states cooker can be in -cookerClean = 1 -cookerParsing = 2 -cookerParsed = 3 - -# Different action states the cooker can be in -cookerRun = 1 # Cooker is running normally -cookerShutdown = 2 # Active tasks should be brought to a controlled stop -cookerStop = 3 # Stop, now! +class state: + initial, parsing, running, shutdown, stop = range(5) #============================================================================# # BBCooker @@ -65,9 +65,7 @@ class BBCooker: def __init__(self, configuration, server): self.status = None - - self.cache = None - self.bb_cache = None + self.appendlist = {} if server: self.server = server.BitBakeServer(self) @@ -102,13 +100,12 @@ class BBCooker: import termios tcattr = termios.tcgetattr(fd) if tcattr[3] & termios.TOSTOP: - bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...") + buildlog.info("The terminal had the TOSTOP bit set, clearing...") tcattr[3] = tcattr[3] & ~termios.TOSTOP termios.tcsetattr(fd, termios.TCSANOW, tcattr) self.command = bb.command.Command(self) - self.cookerState = cookerClean - self.cookerAction = cookerRun + self.state = state.initial def parseConfiguration(self): @@ -118,7 +115,7 @@ class BBCooker: if nice: curnice = os.nice(0) nice = int(nice) - curnice - bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice)) + buildlog.verbose("Renice to %s " % os.nice(nice)) def parseCommandLine(self): # Parse any commandline into actions @@ -126,11 +123,11 @@ class BBCooker: self.commandlineAction = None if 'world' in self.configuration.pkgs_to_build: - bb.msg.error(bb.msg.domain.Build, "'world' is not a valid target for --environment.") + buildlog.error("'world' is not a valid target for --environment.") elif len(self.configuration.pkgs_to_build) > 1: - bb.msg.error(bb.msg.domain.Build, "Only one target can be used with the --environment option.") + buildlog.error("Only one target can be used with the --environment option.") elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0: - bb.msg.error(bb.msg.domain.Build, "No target should be used with the --environment and --buildfile options.") + buildlog.error("No target should be used with the --environment and --buildfile options.") elif len(self.configuration.pkgs_to_build) > 0: self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build] else: @@ -148,13 +145,13 @@ class BBCooker: self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd] else: self.commandlineAction = None - bb.msg.error(bb.msg.domain.Build, "Please specify a package name for dependency graph generation.") + buildlog.error("Please specify a package name for dependency graph generation.") else: if self.configuration.pkgs_to_build: self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd] else: self.commandlineAction = None - bb.msg.error(bb.msg.domain.Build, "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + buildlog.error("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") def runCommands(self, server, data, abort): """ @@ -180,8 +177,8 @@ class BBCooker: preferred_versions[pn] = (pref_ver, pref_file) latest_versions[pn] = (last_ver, last_file) - bb.msg.plain("%-35s %25s %25s" % ("Package Name", "Latest Version", "Preferred Version")) - bb.msg.plain("%-35s %25s %25s\n" % ("============", "==============", "=================")) + logger.plain("%-35s %25s %25s", "Package Name", "Latest Version", "Preferred Version") + logger.plain("%-35s %25s %25s\n", "============", "==============", "=================") for p in sorted(pkg_pn): pref = preferred_versions[p] @@ -193,11 +190,7 @@ class BBCooker: if pref == latest: prefstr = "" - bb.msg.plain("%-35s %25s %25s" % (p, lateststr, prefstr)) - - def compareRevisions(self): - ret = bb.fetch.fetcher_compare_revisons(self.configuration.data) - bb.event.fire(bb.command.CookerCommandSetExitCode(ret), self.configuration.event_data) + logger.plain("%-35s %25s %25s", p, lateststr, prefstr) def showEnvironment(self, buildfile = None, pkgs_to_build = []): """ @@ -207,8 +200,6 @@ class BBCooker: envdata = None if buildfile: - self.cb = None - self.bb_cache = bb.cache.init(self) fn = self.matchFile(buildfile) elif len(pkgs_to_build) == 1: self.updateCache() @@ -229,28 +220,22 @@ class BBCooker: if fn: try: - envdata = self.bb_cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data) - except IOError as e: - bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e)) - raise - except Exception as e: - bb.msg.error(bb.msg.domain.Parsing, "%s" % e) + envdata = bb.cache.Cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data) + except Exception, e: + parselog.exception("Unable to read %s", fn) raise # emit variables and shell functions - try: - data.update_data(envdata) - with closing(StringIO()) as env: - data.emit_env(env, envdata, True) - bb.msg.plain(env.getvalue()) - except Exception as e: - bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e) + data.update_data(envdata) + with closing(StringIO()) as env: + data.emit_env(env, envdata, True) + logger.plain(env.getvalue()) # emit the metadata which isnt valid shell data.expandKeys(envdata) for e in envdata.keys(): if data.getVarFlag( e, 'python', envdata ): - bb.msg.plain("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1))) + logger.plain("\npython %s () {\n%s}\n", e, data.getVar(e, envdata, 1)) def generateDepTreeData(self, pkgs_to_build, task): """ @@ -290,7 +275,7 @@ class BBCooker: depend_tree["rdepends-pkg"] = {} depend_tree["rrecs-pkg"] = {} - for task in range(len(rq.rqdata.runq_fnid)): + for task in xrange(len(rq.rqdata.runq_fnid)): taskname = rq.rqdata.runq_task[task] fnid = rq.rqdata.runq_fnid[task] fn = taskdata.fn_index[fnid] @@ -374,7 +359,7 @@ class BBCooker: for rdepend in depgraph["rdepends-pn"][pn]: print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file) print("}", file=depends_file) - bb.msg.plain("PN dependencies saved to 'pn-depends.dot'") + logger.info("PN dependencies saved to 'pn-depends.dot'") depends_file = file('package-depends.dot', 'w' ) print("digraph depends {", file=depends_file) @@ -395,7 +380,7 @@ class BBCooker: for rdepend in depgraph["rrecs-pkg"][package]: print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file) print("}", file=depends_file) - bb.msg.plain("Package dependencies saved to 'package-depends.dot'") + logger.info("Package dependencies saved to 'package-depends.dot'") tdepends_file = file('task-depends.dot', 'w' ) print("digraph depends {", file=tdepends_file) @@ -407,7 +392,7 @@ class BBCooker: for dep in depgraph["tdepends"][task]: print('"%s" -> "%s"' % (task, dep), file=tdepends_file) print("}", file=tdepends_file) - bb.msg.plain("Task dependencies saved to 'task-depends.dot'") + logger.info("Task dependencies saved to 'task-depends.dot'") def buildDepgraph( self ): all_depends = self.status.all_depends @@ -431,10 +416,10 @@ class BBCooker: try: (providee, provider) = p.split(':') except: - bb.msg.fatal(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p) + providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p) continue if providee in self.status.preferred and self.status.preferred[providee] != provider: - bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee])) + providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.status.preferred[providee]) self.status.preferred[providee] = provider # Calculate priorities for each file @@ -443,8 +428,7 @@ class BBCooker: for collection, pattern, regex, _ in self.status.bbfile_config_priorities: if not regex in matched: - bb.msg.warn(bb.msg.domain.Provider, "No bb files matched BBFILE_PATTERN_%s '%s'" % - (collection, pattern)) + collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) def buildWorldTargetList(self): """ @@ -452,19 +436,19 @@ class BBCooker: """ all_depends = self.status.all_depends pn_provides = self.status.pn_provides - bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"") + parselog.debug(1, "collating packages for \"world\"") for f in self.status.possible_world: terminal = True pn = self.status.pkg_fn[f] for p in pn_provides[pn]: if p.startswith('virtual/'): - bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p)) + parselog.debug(2, "World build skipping %s due to %s provider starting with virtual/", f, p) terminal = False break for pf in self.status.providers[p]: if self.status.pkg_fn[pf] != pn: - bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p)) + parselog.debug(2, "World build skipping %s due to both us and %s providing %s", f, pf, p) terminal = False break if terminal: @@ -478,8 +462,9 @@ class BBCooker: """Drop off into a shell""" try: from bb import shell - except ImportError as details: - bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details ) + except ImportError: + parselog.exception("Interactive mode not available") + sys.exit(1) else: shell.start( self ) @@ -493,70 +478,56 @@ class BBCooker: path, _ = os.path.split(path) def parseConfigurationFiles(self, files): - try: - data = self.configuration.data - - bb.parse.init_parser(data, self.configuration.dump_signatures) - for f in files: - data = bb.parse.handle(f, data) - - layerconf = self._findLayerConf() - if layerconf: - bb.msg.debug(2, bb.msg.domain.Parsing, "Found bblayers.conf (%s)" % layerconf) - data = bb.parse.handle(layerconf, data) - - layers = (bb.data.getVar('BBLAYERS', data, True) or "").split() - - data = bb.data.createCopy(data) - for layer in layers: - bb.msg.debug(2, bb.msg.domain.Parsing, "Adding layer %s" % layer) - bb.data.setVar('LAYERDIR', layer, data) - data = bb.parse.handle(os.path.join(layer, "conf", "layer.conf"), data) + def _parse(f, data, include=False): + try: + return bb.parse.handle(f, data, include) + except (IOError, bb.parse.ParseError) as exc: + parselog.critical("Unable to parse %s: %s" % (f, exc)) + sys.exit(1) - # XXX: Hack, relies on the local keys of the datasmart - # instance being stored in the 'dict' attribute and makes - # assumptions about how variable expansion works, but - # there's no better way to force an expansion of a single - # variable across the datastore today, and this at least - # lets us reference LAYERDIR without having to immediately - # eval all our variables that use it. - for key in data.dict: - if key != "_data": - value = data.getVar(key, False) - if value and "${LAYERDIR}" in value: - data.setVar(key, value.replace("${LAYERDIR}", layer)) + data = self.configuration.data + bb.parse.init_parser(data) + for f in files: + data = _parse(f, data) - bb.data.delVar('LAYERDIR', data) + layerconf = self._findLayerConf() + if layerconf: + parselog.debug(2, "Found bblayers.conf (%s)", layerconf) + data = _parse(layerconf, data) - if not data.getVar("BBPATH", True): - bb.fatal("The BBPATH variable is not set") + layers = (bb.data.getVar('BBLAYERS', data, True) or "").split() - data = bb.parse.handle(os.path.join("conf", "bitbake.conf"), data) + data = bb.data.createCopy(data) + for layer in layers: + parselog.debug(2, "Adding layer %s", layer) + bb.data.setVar('LAYERDIR', layer, data) + data = _parse(os.path.join(layer, "conf", "layer.conf"), data) + data.expandVarref('LAYERDIR') - self.configuration.data = data + bb.data.delVar('LAYERDIR', data) - # Handle any INHERITs and inherit the base class - inherits = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split() - for inherit in inherits: - self.configuration.data = bb.parse.handle(os.path.join('classes', '%s.bbclass' % inherit), self.configuration.data, True ) + if not data.getVar("BBPATH", True): + raise SystemExit("The BBPATH variable is not set") - # Nomally we only register event handlers at the end of parsing .bb files - # We register any handlers we've found so far here... - for var in bb.data.getVar('__BBHANDLERS', self.configuration.data) or []: - bb.event.register(var, bb.data.getVar(var, self.configuration.data)) + data = _parse(os.path.join("conf", "bitbake.conf"), data) - if bb.data.getVar("BB_WORKERCONTEXT", self.configuration.data) is None: - bb.fetch.fetcher_init(self.configuration.data) - bb.codeparser.parser_cache_init(self.configuration.data) + self.configuration.data = data - bb.parse.init_parser(data, self.configuration.dump_signatures) + # Handle any INHERITs and inherit the base class + inherits = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split() + for inherit in inherits: + self.configuration.data = _parse(os.path.join('classes', '%s.bbclass' % inherit), self.configuration.data, True ) - bb.event.fire(bb.event.ConfigParsed(), self.configuration.data) + # Nomally we only register event handlers at the end of parsing .bb files + # We register any handlers we've found so far here... + for var in bb.data.getVar('__BBHANDLERS', self.configuration.data) or []: + bb.event.register(var, bb.data.getVar(var, self.configuration.data)) - except IOError as e: - bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (files, str(e))) - except bb.parse.ParseError as details: - bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (files, details) ) + if bb.data.getVar("BB_WORKERCONTEXT", self.configuration.data) is None: + bb.fetch.fetcher_init(self.configuration.data) + bb.codeparser.parser_cache_init(self.configuration.data) + bb.parse.init_parser(data) + bb.event.fire(bb.event.ConfigParsed(), self.configuration.data) def handleCollections( self, collections ): """Handle collections""" @@ -565,22 +536,22 @@ class BBCooker: for c in collection_list: regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1) if regex == None: - bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c) + parselog.error("BBFILE_PATTERN_%s not defined" % c) continue priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1) if priority == None: - bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c) + parselog.error("BBFILE_PRIORITY_%s not defined" % c) continue try: cre = re.compile(regex) except re.error: - bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex)) + parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex) continue try: pri = int(priority) self.status.bbfile_config_priorities.append((c, regex, cre, pri)) except ValueError: - bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority)) + parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority) def buildSetVars(self): """ @@ -596,7 +567,7 @@ class BBCooker: """ bf = os.path.abspath(buildfile) - (filelist, masked) = self.collect_bbfiles() + filelist, masked = self.collect_bbfiles() try: os.stat(bf) return [bf] @@ -616,9 +587,9 @@ class BBCooker: """ matches = self.matchFiles(buildfile) if len(matches) != 1: - bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches))) + parselog.error("Unable to match %s (%s matches found):" % (buildfile, len(matches))) for f in matches: - bb.msg.error(bb.msg.domain.Parsing, " %s" % f) + parselog.error(" %s" % f) raise MultipleMatches return matches[0] @@ -635,22 +606,23 @@ class BBCooker: if (task == None): task = self.configuration.cmd - self.bb_cache = bb.cache.init(self) - self.status = bb.cache.CacheData() - - (fn, cls) = self.bb_cache.virtualfn2realfn(buildfile) + (fn, cls) = bb.cache.Cache.virtualfn2realfn(buildfile) buildfile = self.matchFile(fn) - fn = self.bb_cache.realfn2virtual(buildfile, cls) + fn = bb.cache.Cache.realfn2virtual(buildfile, cls) self.buildSetVars() - # Load data into the cache for fn and parse the loaded cache data - the_data = self.bb_cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data) - self.bb_cache.setData(fn, buildfile, the_data) - self.bb_cache.handle_data(fn, self.status) + self.status = bb.cache.CacheData() + infos = bb.cache.Cache.parse(fn, self.get_file_appends(fn), \ + self.configuration.data) + maininfo = None + for vfn, info in infos: + self.status.add_from_recipeinfo(vfn, info) + if vfn == fn: + maininfo = info # Tweak some variables - item = self.bb_cache.getVar('PN', fn, True) + item = maininfo.pn self.status.ignored_dependencies = set() self.status.bbfile_priority[fn] = 1 @@ -662,7 +634,7 @@ class BBCooker: # Remove stamp for target if force mode active if self.configuration.force: - bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (task, fn)) + logger.verbose("Remove stamp %s, %s", task, fn) bb.build.del_stamp('do_%s' % task, self.status, fn) # Setup taskdata structure @@ -682,17 +654,17 @@ class BBCooker: def buildFileIdle(server, rq, abort): - if abort or self.cookerAction == cookerStop: + if abort or self.state == state.stop: rq.finish_runqueue(True) - elif self.cookerAction == cookerShutdown: + elif self.state == state.shutdown: rq.finish_runqueue(False) failures = 0 try: retval = rq.execute_runqueue() except runqueue.TaskFailure as exc: for fnid in exc.args: - bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid]) - failures = failures + 1 + buildlog.error("'%s' failed" % taskdata.fn_index[fnid]) + failures += len(exc.args) retval = False if not retval: bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data) @@ -719,17 +691,17 @@ class BBCooker: targets = self.checkPackages(targets) def buildTargetsIdle(server, rq, abort): - if abort or self.cookerAction == cookerStop: + if abort or self.state == state.stop: rq.finish_runqueue(True) - elif self.cookerAction == cookerShutdown: + elif self.state == state.shutdown: rq.finish_runqueue(False) failures = 0 try: retval = rq.execute_runqueue() except runqueue.TaskFailure as exc: for fnid in exc.args: - bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid]) - failures = failures + 1 + buildlog.error("'%s' failed" % taskdata.fn_index[fnid]) + failures += len(exc.args) retval = False if not retval: bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.event_data) @@ -764,12 +736,10 @@ class BBCooker: self.server.register_idle_function(buildTargetsIdle, rq) def updateCache(self): - - if self.cookerState == cookerParsed: + if self.state == state.running: return - if self.cookerState != cookerParsing: - + if self.state != state.parsing: self.parseConfiguration () # Import Psyco if available and not disabled @@ -779,11 +749,11 @@ class BBCooker: try: import psyco except ImportError: - bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.") + collectlog.info("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.") else: psyco.bind( CookerParser.parse_next ) else: - bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.") + collectlog.info("You have disabled Psyco. This decreases performance.") self.status = bb.cache.CacheData() @@ -799,12 +769,12 @@ class BBCooker: bb.data.renameVar("__depends", "__base_depends", self.configuration.data) self.parser = CookerParser(self, filelist, masked) - self.cookerState = cookerParsing + self.state = state.parsing if not self.parser.parse_next(): - bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete") + collectlog.debug(1, "parsing complete") self.buildDepgraph() - self.cookerState = cookerParsed + self.state = state.running return None return True @@ -848,9 +818,8 @@ class BBCooker: def collect_bbfiles( self ): """Collect all available .bb build files""" parsed, cached, skipped, masked = 0, 0, 0, 0 - self.bb_cache = bb.cache.init(self) - bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files") + collectlog.debug(1, "collecting .bb files") files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split() data.setVar("BBFILES", " ".join(files), self.configuration.data) @@ -859,7 +828,7 @@ class BBCooker: files = self.get_bbfiles() if not len(files): - bb.msg.error(bb.msg.domain.Collection, "no recipe files to build, check your BBPATH and BBFILES?") + collectlog.error("no recipe files to build, check your BBPATH and BBFILES?") bb.event.fire(CookerExit(), self.configuration.event_data) newfiles = set() @@ -879,13 +848,14 @@ class BBCooker: try: bbmask_compiled = re.compile(bbmask) except sre_constants.error: - bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.") + collectlog.critical("BBMASK is not a valid regular expression, ignoring.") + return list(newfiles), 0 bbfiles = [] bbappend = [] for f in newfiles: if bbmask and bbmask_compiled.search(f): - bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f) + collectlog.debug(1, "skipping masked file %s", f) masked += 1 continue if f.endswith('.bb'): @@ -893,26 +863,25 @@ class BBCooker: elif f.endswith('.bbappend'): bbappend.append(f) else: - bb.msg.note(1, bb.msg.domain.Collection, "File %s of unknown filetype in BBFILES? Ignorning..." % f) + collectlog.debug(1, "skipping %s: unknown file extension", f) # Build a list of .bbappend files for each .bb file - self.appendlist = {} for f in bbappend: base = os.path.basename(f).replace('.bbappend', '.bb') if not base in self.appendlist: self.appendlist[base] = [] self.appendlist[base].append(f) - + return (bbfiles, masked) def get_file_appends(self, fn): """ Returns a list of .bbappend files to apply to fn - NB: collect_files() must have been called prior to this + NB: collect_bbfiles() must have been called prior to this """ f = os.path.basename(fn) if f in self.appendlist: - return self.appendlist[f] + return self.appendlist[f] return [] def pre_serve(self): @@ -924,6 +893,11 @@ class BBCooker: def post_serve(self): bb.event.fire(CookerExit(), self.configuration.event_data) + def shutdown(self): + self.state = state.shutdown + + def stop(self): + self.state = state.stop def server_main(cooker, func, *args): cooker.pre_serve() @@ -974,65 +948,119 @@ class CookerExit(bb.event.Event): def __init__(self): bb.event.Event.__init__(self) -class CookerParser: +def parse_file(task): + filename, appends = task + try: + return True, bb.cache.Cache.parse(filename, appends, parse_file.cfg) + except Exception, exc: + exc.recipe = filename + raise exc + +class CookerParser(object): def __init__(self, cooker, filelist, masked): - # Internal data self.filelist = filelist self.cooker = cooker + self.cfgdata = cooker.configuration.data # Accounting statistics self.parsed = 0 self.cached = 0 self.error = 0 self.masked = masked - self.total = len(filelist) self.skipped = 0 self.virtuals = 0 + self.total = len(filelist) - # Pointer to the next file to parse - self.pointer = 0 + self.current = 0 + self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or + multiprocessing.cpu_count()) + + self.bb_cache = bb.cache.Cache(self.cfgdata) + self.fromcache = [] + self.willparse = [] + for filename in self.filelist: + appends = self.cooker.get_file_appends(filename) + if not self.bb_cache.cacheValid(filename): + self.willparse.append((filename, appends)) + else: + self.fromcache.append((filename, appends)) + self.toparse = self.total - len(self.fromcache) + self.progress_chunk = max(self.toparse / 100, 1) - def parse_next(self): - cooker = self.cooker - if self.pointer < len(self.filelist): - f = self.filelist[self.pointer] + self.start() - try: - fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.get_file_appends(f), cooker.configuration.data, cooker.status) - if fromCache: - self.cached += 1 - else: - self.parsed += 1 - - self.skipped += skipped - self.virtuals += virtuals - - except IOError as e: - self.error += 1 - cooker.bb_cache.remove(f) - bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e)) - pass - except KeyboardInterrupt: - cooker.bb_cache.remove(f) - cooker.bb_cache.sync() - raise - except Exception as e: - self.error += 1 - cooker.bb_cache.remove(f) - bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f)) - except: - cooker.bb_cache.remove(f) - raise - finally: - bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, self.skipped, self.masked, self.virtuals, self.error, self.total), cooker.configuration.event_data) + def start(self): + def init(cfg): + signal.signal(signal.SIGINT, signal.SIG_IGN) + parse_file.cfg = cfg + + bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) - self.pointer += 1 + self.pool = multiprocessing.Pool(self.num_processes, init, [self.cfgdata]) + parsed = self.pool.imap(parse_file, self.willparse) + self.pool.close() - if self.pointer >= self.total: - cooker.bb_cache.sync() - bb.codeparser.parser_cache_save(cooker.configuration.data) - if self.error > 0: - raise ParsingErrorsFound + self.results = itertools.chain(self.load_cached(), parsed) + + def shutdown(self, clean=True): + if clean: + event = bb.event.ParseCompleted(self.cached, self.parsed, + self.skipped, self.masked, + self.virtuals, self.error, + self.total) + bb.event.fire(event, self.cfgdata) + else: + self.pool.terminate() + self.pool.join() + + sync = threading.Thread(target=self.bb_cache.sync) + sync.start() + atexit.register(lambda: sync.join()) + + codesync = threading.Thread(target=bb.codeparser.parser_cache_save(self.cooker.configuration.data)) + codesync.start() + atexit.register(lambda: codesync.join()) + + def load_cached(self): + for filename, appends in self.fromcache: + cached, infos = self.bb_cache.load(filename, appends, self.cfgdata) + yield not cached, infos + + def parse_next(self): + try: + parsed, result = self.results.next() + except StopIteration: + self.shutdown() return False + except KeyboardInterrupt: + self.shutdown(clean=False) + raise + except Exception as exc: + self.shutdown(clean=False) + bb.fatal('Error parsing %s: %s' % (exc.recipe, exc)) + + self.current += 1 + self.virtuals += len(result) + if parsed: + self.parsed += 1 + if self.parsed % self.progress_chunk == 0: + bb.event.fire(bb.event.ParseProgress(self.parsed), + self.cfgdata) + else: + self.cached += 1 + + for virtualfn, info in result: + if info.skipped: + self.skipped += 1 + else: + self.bb_cache.add_info(virtualfn, info, self.cooker.status, + parsed=parsed) return True + + def reparse(self, filename): + infos = self.bb_cache.parse(filename, + self.cooker.get_file_appends(filename), + self.cfgdata) + for vfn, info in infos: + self.cooker.status.add_from_recipeinfo(vfn, info) |