cache: sync the cache file to disk in the background

This version uses a thread rather than a process, to avoid problems with
waitpid handling.  This gives slightly less overall build time reduction than
the separate process for it did (this reduces a -c compile coreutils-native by
about 3 seconds, while the process reduced it by 7 seconds), however this time
is quite insignificant relative to a typical build.

The biggest issue with non-backgrounded syncing is the perceived delay before
work begins, and this resolves that without breaking anything, or so it seems.

(Bitbake rev: 5ab6c5c7b007b8c77c751582141afc07c183d672)

Signed-off-by: Chris Larson <chris_larson@mentor.com>
Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

4 files changed, 60 insertions, 43 deletions

diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 0143c149b8..ac9758d402 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -25,6 +25,7 @@ from __future__ import print_function
 import sys, os, glob, os.path, re, time
 import logging
 import sre_constants
+import threading
 import multiprocessing
 import signal
 from cStringIO import StringIO
@@ -983,6 +984,7 @@ class CookerParser(object):
         # Internal data
         self.filelist = filelist
         self.cooker = cooker
+        self.cfgdata = cooker.configuration.data
 
         # Accounting statistics
         self.parsed = 0
@@ -1006,7 +1008,6 @@ class CookerParser(object):
         self.result_queue = multiprocessing.Queue()
  
         self.fromcache = []
-        cfgdata = self.cooker.configuration.data
         for filename in self.filelist:
             appends = self.cooker.get_file_appends(filename)
             if not self.cooker.bb_cache.cacheValid(filename):
@@ -1021,13 +1022,13 @@ class CookerParser(object):
                 output.put(infos)
 
         self.processes = []
-        num_processes = int(cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
+        num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
                             multiprocessing.cpu_count())
         for i in xrange(num_processes):
             process = multiprocessing.Process(target=worker,
                                               args=(self.task_queue,
                                                     self.result_queue,
-                                                    cfgdata))
+                                                    self.cfgdata))
             process.start()
             self.processes.append(process)
 
@@ -1041,29 +1042,29 @@ class CookerParser(object):
         self.task_queue.close()
         for process in self.processes:
             process.join()
-        self.cooker.bb_cache.sync()
-        bb.codeparser.parser_cache_save(self.cooker.configuration.data)
+        threading.Thread(target=self.cooker.bb_cache.sync).start()
+        threading.Thread(target=bb.codeparser.parser_cache_save(self.cooker.configuration.data)).start()
         if self.error > 0:
             raise ParsingErrorsFound()
 
-    def progress(self):
-        bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed,
-                                                self.skipped, self.masked,
-                                                self.virtuals, self.error,
-                                                self.total),
-                        self.cooker.configuration.event_data)
-
     def parse_next(self):
         cooker = self.cooker
         if self.current >= self.total:
+            event = bb.event.ParseCompleted(self.cached, self.parsed,
+                                            self.skipped, self.masked,
+                                            self.virtuals, self.error,
+                                            self.total)
+            bb.event.fire(event, self.cfgdata)
             self.shutdown()
             return False
+        elif self.current == 0:
+            bb.event.fire(bb.event.ParseStarted(self.total, self.skipped, self.masked),
+                          self.cfgdata)
 
         try:
             if self.result_queue.empty() and self.fromcache:
                 filename, appends = self.fromcache.pop()
-                _, infos = cooker.bb_cache.load(filename, appends,
-                                                self.cooker.configuration.data)
+                _, infos = cooker.bb_cache.load(filename, appends, self.cfgdata)
                 parsed = False
             else:
                 infos = self.result_queue.get()
@@ -1087,7 +1088,7 @@ class CookerParser(object):
                 if info.skipped:
                     self.skipped += 1
         finally:
-            self.progress()
+            bb.event.fire(bb.event.ParseProgress(self.current), self.cfgdata)
 
         self.current += 1
         return True
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 5f292bccbc..5b0a183acd 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -296,10 +296,16 @@ class MultipleProviders(Event):
         """
         return self._candidates
 
-class ParseProgress(Event):
-    """
-    Parsing Progress Event
-    """
+class ParseStarted(Event):
+    """Recipe parsing for the runqueue has begun"""
+    def __init__(self, total, skipped, masked):
+        Event.__init__(self)
+        self.total = total
+        self.skipped = skipped
+        self.masked = masked
+
+class ParseCompleted(Event):
+    """Recipe parsing for the runqueue has completed"""
 
     def __init__(self, cached, parsed, skipped, masked, virtuals, errors, total):
         Event.__init__(self)
@@ -312,6 +318,12 @@ class ParseProgress(Event):
         self.sofar = cached + parsed
         self.total = total
 
+class ParseProgress(Event):
+    """Recipe parsing progress"""
+
+    def __init__(self, current):
+        self.current = current
+
 class DepTreeGenerated(Event):
     """
     Event when a dependency tree has been generated
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index a34991bb68..326f164353 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -78,6 +78,7 @@ def init(server, eventHandler):
         return 1
 
     pbar = None
+    interactive = os.isatty(sys.stdout.fileno())
     shutdown = 0
     return_value = 0
     while True:
@@ -132,23 +133,26 @@ def init(server, eventHandler):
             if isinstance(event, bb.build.TaskBase):
                 logger.info(event._message)
                 continue
+            if isinstance(event, bb.event.ParseStarted):
+                if interactive:
+                    pbar = progressbar.ProgressBar(widgets=widgets,
+                                                   maxval=event.total).start()
+                else:
+                    sys.stdout.write("Parsing recipes...")
+                    sys.stdout.flush()
+                continue
             if isinstance(event, bb.event.ParseProgress):
-                current, total = event.sofar, event.total
-                if os.isatty(sys.stdout.fileno()):
-                    if not pbar:
-                        pbar = progressbar.ProgressBar(widgets=widgets,
-                                                       maxval=total).start()
-                    pbar.update(current)
+                if interactive:
+                    pbar.update(event.current)
+                continue
+            if isinstance(event, bb.event.ParseCompleted):
+                if interactive:
+                    pbar.update(event.total)
                 else:
-                    if current == 1:
-                        sys.stdout.write("Parsing .bb files, please wait...")
-                        sys.stdout.flush()
-                    if current == total:
-                        sys.stdout.write("done.")
-                        sys.stdout.flush()
-                if current == total:
-                    print(("\nParsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
-                        % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
+                    sys.stdout.write("done.\n")
+                    sys.stdout.flush()
+                print(("\nParsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
+                    % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
                 continue
 
             if isinstance(event, bb.command.CookerCommandCompleted):
diff --git a/bitbake/lib/bb/ui/uievent.py b/bitbake/lib/bb/ui/uievent.py
index f1e4d791ee..b404805d81 100644
--- a/bitbake/lib/bb/ui/uievent.py
+++ b/bitbake/lib/bb/ui/uievent.py
@@ -37,8 +37,8 @@ class BBUIEventQueue:
         self.BBServer = BBServer
 
         self.t = threading.Thread()
-        self.t.setDaemon(True)

-        self.t.run = self.startCallbackHandler

+        self.t.setDaemon(True)
+        self.t.run = self.startCallbackHandler
         self.t.start()
 
     def getEvent(self):
@@ -70,7 +70,7 @@ class BBUIEventQueue:
     def startCallbackHandler(self):
 
         server = UIXMLRPCServer()
-        self.host, self.port = server.socket.getsockname()

+        self.host, self.port = server.socket.getsockname()
 
         server.register_function( self.system_quit, "event.quit" )
         server.register_function( self.queue_event, "event.send" )
@@ -83,7 +83,7 @@ class BBUIEventQueue:
             server.handle_request()
         server.server_close()
 
-    def system_quit( self ):

+    def system_quit( self ):
         """
         Shut down the callback thread
         """
@@ -95,11 +95,11 @@ class BBUIEventQueue:
 
 class UIXMLRPCServer (SimpleXMLRPCServer):
 
-    def __init__( self, interface = ("localhost", 0) ):

+    def __init__( self, interface = ("localhost", 0) ):
         self.quit = False
-        SimpleXMLRPCServer.__init__( self,

-                                    interface,

-                                    requestHandler=SimpleXMLRPCRequestHandler,

+        SimpleXMLRPCServer.__init__( self,
+                                    interface,
+                                    requestHandler=SimpleXMLRPCRequestHandler,
                                     logRequests=False, allow_none=True)
 
     def get_request(self):
@@ -121,4 +121,4 @@ class UIXMLRPCServer (SimpleXMLRPCServer):
         if request is None:
             return
         SimpleXMLRPCServer.process_request(self, request, client_address)
-

+