From 9747616600b424e8c0d7075a9b516a12d65a17c3 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Tue, 28 Sep 2010 22:24:13 +0100 Subject: bitbake/siggen: Ensure full signature data is not held unless needed, reducing memory consumption Signed-off-by: Richard Purdie --- bitbake/lib/bb/cooker.py | 4 ++-- bitbake/lib/bb/parse/__init__.py | 4 ++-- bitbake/lib/bb/siggen.py | 29 ++++++++++++++++++----------- 3 files changed, 22 insertions(+), 15 deletions(-) (limited to 'bitbake') diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 5cea9dba9d..44b9b2c31a 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -497,7 +497,7 @@ class BBCooker: try: data = self.configuration.data - bb.parse.init_parser(data) + bb.parse.init_parser(data, self.configuration.dump_signatures) for f in files: data = bb.parse.handle(f, data) @@ -550,7 +550,7 @@ class BBCooker: bb.fetch.fetcher_init(self.configuration.data) bb.codeparser.parser_cache_init(self.configuration.data) - bb.parse.init_parser(data) + bb.parse.init_parser(data, self.configuration.dump_signatures) bb.event.fire(bb.event.ConfigParsed(), self.configuration.data) diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py index c7249ef050..da160ceb27 100644 --- a/bitbake/lib/bb/parse/__init__.py +++ b/bitbake/lib/bb/parse/__init__.py @@ -80,8 +80,8 @@ def init(fn, data): if h['supports'](fn): return h['init'](data) -def init_parser(d): - bb.parse.siggen = bb.siggen.init(d) +def init_parser(d, dumpsigs): + bb.parse.siggen = bb.siggen.init(d, dumpsigs) def resolve_file(fn, d): if not os.path.isabs(fn): diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index eb624311a0..2948372c6a 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -7,26 +7,26 @@ except ImportError: import pickle bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.") -def init(d): +def init(d, dumpsigs): siggens = [obj for obj in globals().itervalues() if type(obj) is type and issubclass(obj, SignatureGenerator)] desired = bb.data.getVar("BB_SIGNATURE_HANDLER", d, True) or "noop" for sg in siggens: if desired == sg.name: - return sg(d) + return sg(d, dumpsigs) break else: bb.error("Invalid signature generator '%s', using default 'noop' generator" % desired) bb.error("Available generators: %s" % ", ".join(obj.name for obj in siggens)) - return SignatureGenerator(d) + return SignatureGenerator(d, dumpsigs) class SignatureGenerator(object): """ """ name = "noop" - def __init__(self, data): + def __init__(self, data, dumpsigs): return def finalise(self, fn, d): @@ -37,7 +37,7 @@ class SignatureGeneratorBasic(SignatureGenerator): """ name = "basic" - def __init__(self, data): + def __init__(self, data, dumpsigs): self.basehash = {} self.taskhash = {} self.taskdeps = {} @@ -52,17 +52,19 @@ class SignatureGeneratorBasic(SignatureGenerator): else: self.twl = None + self.dumpsigs = dumpsigs + def _build_data(self, fn, d): - self.taskdeps[fn], self.gendeps[fn] = bb.data.generate_dependencies(d) + taskdeps, gendeps = bb.data.generate_dependencies(d) basehash = {} lookupcache = {} - for task in self.taskdeps[fn]: + for task in taskdeps: data = d.getVar(task, False) lookupcache[task] = data - for dep in sorted(self.taskdeps[fn][task]): + for dep in sorted(taskdeps[task]): if dep in self.basewhitelist: continue if dep in lookupcache: @@ -75,20 +77,25 @@ class SignatureGeneratorBasic(SignatureGenerator): self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest() #bb.note("Hash for %s is %s" % (task, tashhash[task])) - self.lookupcache[fn] = lookupcache + if self.dumpsigs: + self.taskdeps[fn] = taskdeps + self.gendeps[fn] = gendeps + self.lookupcache[fn] = lookupcache + + return taskdeps def finalise(self, fn, d, variant): if variant: fn = "virtual:" + variant + ":" + fn - self._build_data(fn, d) + taskdeps = self._build_data(fn, d) #Slow but can be useful for debugging mismatched basehashes #for task in self.taskdeps[fn]: # self.dump_sigtask(fn, task, d.getVar("STAMP", True), False) - for task in self.taskdeps[fn]: + for task in taskdeps: d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) def get_taskhash(self, fn, task, deps, dataCache): -- cgit v1.2.3