bitbake: Upgrade from 1.4 -> 1.7.4ish

git-svn-id: https://svn.o-hand.com/repos/poky/trunk@863 311d38ba-8fff-0310-9ca6-ca027cbcb966

44 files changed, 3720 insertions, 1809 deletions

diff --git a/bitbake/ChangeLog b/bitbake/ChangeLog
index 93cc45aaf4..135aba9fee 100644
--- a/bitbake/ChangeLog
+++ b/bitbake/ChangeLog
@@ -1,3 +1,21 @@
+Changes in BitBake 1.7.3:
+
+Changes in BitBake 1.7.1:
+    - Major updates of the dependency handling and execution
+      of tasks
+    - Change of the SVN Fetcher to keep the checkout around
+      courtsey to Paul Sokolovsky (#1367)
+
+Changes in Bitbake 1.6.0:
+	- Better msg handling
+	- COW dict implementation from Tim Ansell (mithro) leading
+	  to better performance
+	- Speed up of -s
+
+Changes in Bitbake 1.4.4:
+	- SRCDATE now handling courtsey Justin Patrin
+	- #1017 fix to work with rm_work
+
 Changes in BitBake 1.4.2:
 	- Send logs to oe.pastebin.com instead of pastebin.com
 	  fixes #856
diff --git a/bitbake/MANIFEST b/bitbake/MANIFEST
index 144c74c0fe..f088792aa4 100644
--- a/bitbake/MANIFEST
+++ b/bitbake/MANIFEST
@@ -5,29 +5,34 @@ setup.py
 bin/bitdoc
 bin/bbimage
 bin/bitbake
+lib/bb/COW.py
 lib/bb/__init__.py
 lib/bb/build.py
 lib/bb/cache.py
 lib/bb/data.py
 lib/bb/data_smart.py
 lib/bb/event.py
-lib/bb/fetch/bk.py
+lib/bb/manifest.py
+lib/bb/methodpool.py
+lib/bb/msg.py
+lib/bb/providers.py
+lib/bb/runqueue.py
+lib/bb/shell.py
+lib/bb/taskdata.py
+lib/bb/utils.py
 lib/bb/fetch/cvs.py
 lib/bb/fetch/git.py
 lib/bb/fetch/__init__.py
 lib/bb/fetch/local.py
+lib/bb/fetch/perforce.py
+lib/bb/fetch/ssh.py
 lib/bb/fetch/svk.py
 lib/bb/fetch/svn.py
 lib/bb/fetch/wget.py
-lib/bb/fetch/ssh.py
-lib/bb/manifest.py
-lib/bb/methodpool.py
 lib/bb/parse/__init__.py
 lib/bb/parse/parse_py/BBHandler.py
 lib/bb/parse/parse_py/ConfHandler.py
 lib/bb/parse/parse_py/__init__.py
-lib/bb/shell.py
-lib/bb/utils.py
 doc/COPYING.GPL
 doc/COPYING.MIT
 doc/manual/html.css
diff --git a/bitbake/TODO b/bitbake/TODO
deleted file mode 100644
index 511fae4a25..0000000000
--- a/bitbake/TODO
+++ /dev/null
@@ -1,18 +0,0 @@
-On popular request by popular people a list of tasks to-do:
-
-	-Kill insecure usage of os.system either by properly escaping
-         the strings or a faster replacement not involving /bin/sh
-	-Introduce a -p option to automatically hotshot/profile the
-	 run
-	-Cache dependencies separately and invalidate them when any file
-         changed.
-	-...
-
-
-DONE:
-·       -On generating the inter package deps do not parse each file multiply
-·        times.
-	-We build the lists while parsing the data now
-·       (WAS: Do not generate the world dependency tree, only when someone
-·             requests it.
-
diff --git a/bitbake/bin/bbimage b/bitbake/bin/bbimage
index df6caa28ed..9adedbfc63 100755
--- a/bitbake/bin/bbimage
+++ b/bitbake/bin/bbimage
@@ -18,15 +18,16 @@
 # Place, Suite 330, Boston, MA 02111-1307 USA. 
 
 import sys, os
-sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 import bb
 from bb import *
 
-__version__ = 1.0
+__version__ = 1.1
 type = "jffs2"
 cfg_bb = data.init()
 cfg_oespawn = data.init()
 
+bb.msg.set_debug_level(0)
 
 def usage():
     print "Usage: bbimage [options ...]"
diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
index 7fbe7ed5eb..85a0cbc398 100755
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -7,6 +7,7 @@
 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
 # Copyright (C) 2005        Holger Hans Peter Freyther
 # Copyright (C) 2005        ROAD GmbH
+# Copyright (C) 2006        Richard Purdie
 #
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software
@@ -24,136 +25,13 @@
 import sys, os, getopt, glob, copy, os.path, re, time
 sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 import bb
-from bb import utils, data, parse, debug, event, fatal, cache
+from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
 from sets import Set
 import itertools, optparse
 
 parsespin = itertools.cycle( r'|/-\\' )
-bbdebug = 0
-
-__version__ = "1.4.3"
-
-#============================================================================#
-# BBParsingStatus
-#============================================================================#
-class BBParsingStatus:
-    """
-    The initial idea for this status class is to use the data when it is
-    already loaded instead of loading it from various place over and over
-    again.
-    """
-
-    def __init__(self):
-        self.providers   = {}
-        self.rproviders = {}
-        self.packages = {}
-        self.packages_dynamic = {}
-        self.bbfile_priority = {}
-        self.bbfile_config_priorities = []
-        self.ignored_dependencies = None
-        self.possible_world = []
-        self.world_target = Set()
-        self.pkg_pn = {}
-        self.pkg_fn = {}
-        self.pkg_pvpr = {}
-        self.pkg_dp = {}
-        self.pn_provides = {}
-        self.all_depends = Set()
-        self.build_all = {}
-        self.rundeps = {}
-        self.runrecs = {}
-        self.stamp = {}
-
-    def handle_bb_data(self, file_name, bb_cache, cached):
-        """
-        We will fill the dictionaries with the stuff we
-        need for building the tree more fast
-        """
-
-        pn       = bb_cache.getVar('PN', file_name, True)
-        pv       = bb_cache.getVar('PV', file_name, True)
-        pr       = bb_cache.getVar('PR', file_name, True)
-        dp       = int(bb_cache.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
-        provides  = Set([pn] + (bb_cache.getVar("PROVIDES", file_name, True) or "").split())
-        depends   = (bb_cache.getVar("DEPENDS", file_name, True) or "").split()
-        packages  = (bb_cache.getVar('PACKAGES', file_name, True) or "").split()
-        packages_dynamic = (bb_cache.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
-        rprovides = (bb_cache.getVar("RPROVIDES", file_name, True) or "").split()
-
-        # build PackageName to FileName lookup table
-        if pn not in self.pkg_pn:
-            self.pkg_pn[pn] = []
-        self.pkg_pn[pn].append(file_name)
-
-        self.build_all[file_name] = int(bb_cache.getVar('BUILD_ALL_DEPS', file_name, True) or "0")
-        self.stamp[file_name] = bb_cache.getVar('STAMP', file_name, True)
-
-        # build FileName to PackageName lookup table
-        self.pkg_fn[file_name] = pn
-        self.pkg_pvpr[file_name] = (pv,pr)
-        self.pkg_dp[file_name] = dp
-
-        # Build forward and reverse provider hashes
-        # Forward: virtual -> [filenames]
-        # Reverse: PN -> [virtuals]
-        if pn not in self.pn_provides:
-            self.pn_provides[pn] = Set()
-        self.pn_provides[pn] |= provides
-
-        for provide in provides:
-            if provide not in self.providers:
-                self.providers[provide] = []
-            self.providers[provide].append(file_name)
-
-        for dep in depends:
-            self.all_depends.add(dep)
-
-        # Build reverse hash for PACKAGES, so runtime dependencies 
-        # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
-        for package in packages:
-            if not package in self.packages:
-                self.packages[package] = []
-            self.packages[package].append(file_name)
-            rprovides += (bb_cache.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() 
-
-        for package in packages_dynamic:
-            if not package in self.packages_dynamic:
-                self.packages_dynamic[package] = []
-            self.packages_dynamic[package].append(file_name)
-
-        for rprovide in rprovides:
-            if not rprovide in self.rproviders:
-                self.rproviders[rprovide] = []
-            self.rproviders[rprovide].append(file_name)
-
-        # Build hash of runtime depeneds and rececommends
-
-        def add_dep(deplist, deps):
-            for dep in deps:
-                if not dep in deplist:
-                    deplist[dep] = ""
-
-        if not file_name in self.rundeps:
-            self.rundeps[file_name] = {}
-        if not file_name in self.runrecs:
-            self.runrecs[file_name] = {}
-
-        for package in packages + [pn]:
-            if not package in self.rundeps[file_name]:
-                self.rundeps[file_name][package] = {}
-            if not package in self.runrecs[file_name]:
-                self.runrecs[file_name][package] = {}
-
-            add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RDEPENDS', file_name, True) or ""))
-            add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RRECOMMENDS', file_name, True) or ""))
-            add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
-            add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
-
-        # Collect files we may need for possible world-dep
-        # calculations
-        if not bb_cache.getVar('BROKEN', file_name, True) and not bb_cache.getVar('EXCLUDE_FROM_WORLD', file_name, True):
-            self.possible_world.append(file_name)
 
+__version__ = "1.7.4"
 
 #============================================================================#
 # BBStatistics
@@ -198,207 +76,63 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    ParsingStatus = BBParsingStatus     # make it visible from the shell
     Statistics = BBStatistics           # make it visible from the shell
 
     def __init__( self ):
         self.build_cache_fail = []
         self.build_cache = []
-        self.rbuild_cache = []
-        self.building_list = []
-        self.build_path = []
-        self.consider_msgs_cache = []
-        self.preferred = {}
         self.stats = BBStatistics()
         self.status = None
 
         self.cache = None
         self.bb_cache = None
 
-    def tryBuildPackage( self, fn, item, the_data ):
-        """Build one package"""
+    def tryBuildPackage(self, fn, item, task, the_data, build_depends):
+        """
+        Build one task of a package, optionally build following task depends
+        """
         bb.event.fire(bb.event.PkgStarted(item, the_data))
         try:
             self.stats.attempt += 1
             if self.configuration.force:
-                bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data)
+                bb.data.setVarFlag('do_%s' % task, 'force', 1, the_data)
+            if not build_depends:
+                bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
             if not self.configuration.dry_run:
-                bb.build.exec_task('do_%s' % self.configuration.cmd, the_data)
+                bb.build.exec_task('do_%s' % task, the_data)
             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
             self.build_cache.append(fn)
             return True
         except bb.build.FuncFailed:
             self.stats.fail += 1
-            bb.error("task stack execution failed")
+            bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
             bb.event.fire(bb.event.PkgFailed(item, the_data))
             self.build_cache_fail.append(fn)
             raise
         except bb.build.EventException, e:
             self.stats.fail += 1
             event = e.args[1]
-            bb.error("%s event exception, aborting" % bb.event.getName(event))
+            bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
             bb.event.fire(bb.event.PkgFailed(item, the_data))
             self.build_cache_fail.append(fn)
             raise
 
-    def tryBuild( self, fn, virtual , buildAllDeps , build_depends = []):
+    def tryBuild( self, fn, build_depends):
         """
         Build a provider and its dependencies. 
         build_depends is a list of previous build dependencies (not runtime)
         If build_depends is empty, we're dealing with a runtime depends
         """
 
-        the_data = self.bb_cache.loadDataFull(fn, self)
-
-        # Only follow all (runtime) dependencies if doing a build
-        if not buildAllDeps and self.configuration.cmd is "build":
-            buildAllDeps = self.status.build_all[fn]
-
-        # Error on build time dependency loops
-        if build_depends and build_depends.count(fn) > 1:
-            bb.error("%s depends on itself (eventually)" % fn)
-            bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
-            return False
-
-        # See if this is a runtime dependency we've already built
-	# Or a build dependency being handled in a different build chain
-        if fn in self.building_list:
-            return self.addRunDeps(fn, virtual , buildAllDeps)
+        the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
 
         item = self.status.pkg_fn[fn]
 
-        self.building_list.append(fn)
-
-        pathstr = "%s (%s)" % (item, virtual)
-        self.build_path.append(pathstr)
-
-        depends_list = (bb.data.getVar('DEPENDS', the_data, True) or "").split()
-
-        if self.configuration.verbose:
-            bb.note("current path: %s" % (" -> ".join(self.build_path)))
-            bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
-
-        try:
-            failed = False
-
-            depcmd = self.configuration.cmd
-            bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data)
-            if bbdepcmd is not None:
-                if bbdepcmd == "":
-                    depcmd = None
-                else:
-                    depcmd = bbdepcmd
-
-            if depcmd:
-                oldcmd = self.configuration.cmd
-                self.configuration.cmd = depcmd
-
-            for dependency in depends_list:
-                if dependency in self.status.ignored_dependencies:
-                    continue
-                if not depcmd:
-                    continue
-                if self.buildProvider( dependency , buildAllDeps , build_depends ) == 0:
-                    bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
-                    failed = True
-                    if self.configuration.abort:
-                        break
-
-            if depcmd:
-                self.configuration.cmd = oldcmd
-
-            if failed:
-                self.stats.deps += 1
-                return False
-
-            if not self.addRunDeps(fn, virtual , buildAllDeps):
-                return False
-
-            if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
-                self.build_cache.append(fn)
-                return True
-
-            return self.tryBuildPackage( fn, item, the_data )
-
-        finally:
-            self.building_list.remove(fn)
-            self.build_path.remove(pathstr)
-
-    def findBestProvider( self, pn, pkg_pn = None):
-        """
-        If there is a PREFERRED_VERSION, find the highest-priority bbfile
-        providing that version.  If not, find the latest version provided by
-        an bbfile in the highest-priority set.
-        """
-        if not pkg_pn:
-            pkg_pn = self.status.pkg_pn
-
-        files = pkg_pn[pn]
-        priorities = {}
-        for f in files:
-            priority = self.status.bbfile_priority[f]
-            if priority not in priorities:
-                priorities[priority] = []
-            priorities[priority].append(f)
-        p_list = priorities.keys()
-        p_list.sort(lambda a, b: a - b)
-        tmp_pn = []
-        for p in p_list:
-            tmp_pn = [priorities[p]] + tmp_pn
-
-        preferred_file = None
-
-        localdata = data.createCopy(self.configuration.data)
-        bb.data.setVar('OVERRIDES', "%s:%s" % (pn, data.getVar('OVERRIDES', localdata)), localdata)
-        bb.data.update_data(localdata)
-
-        preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True)
-        if preferred_v:
-            m = re.match('(.*)_(.*)', preferred_v)
-            if m:
-                preferred_v = m.group(1)
-                preferred_r = m.group(2)
-            else:
-                preferred_r = None
-
-            for file_set in tmp_pn:
-                for f in file_set:
-                    pv,pr = self.status.pkg_pvpr[f]
-                    if preferred_v == pv and (preferred_r == pr or preferred_r == None):
-                        preferred_file = f
-                        preferred_ver = (pv, pr)
-                        break
-                if preferred_file:
-                    break;
-            if preferred_r:
-                pv_str = '%s-%s' % (preferred_v, preferred_r)
-            else:
-                pv_str = preferred_v
-            if preferred_file is None:
-                bb.note("preferred version %s of %s not available" % (pv_str, pn))
-            else:
-                bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
-
-        del localdata
-
-        # get highest priority file set
-        files = tmp_pn[0]
-        latest = None
-        latest_p = 0
-        latest_f = None
-        for file_name in files:
-            pv,pr = self.status.pkg_pvpr[file_name]
-            dp = self.status.pkg_dp[file_name]
-
-            if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
-                latest = (pv, pr)
-                latest_f = file_name
-                latest_p = dp
-        if preferred_file is None:
-            preferred_file = latest_f
-            preferred_ver = latest
+        if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data) and not self.configuration.force:
+            self.build_cache.append(fn)
+            return True
 
-        return (latest,latest_f,preferred_ver, preferred_file)
+        return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
 
     def showVersions( self ):
         pkg_pn = self.status.pkg_pn
@@ -407,7 +141,7 @@ class BBCooker:
 
         # Sort by priority
         for pn in pkg_pn.keys():
-            (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
+            (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
             preferred_versions[pn] = (pref_ver, pref_file)
             latest_versions[pn] = (last_ver, last_file)
 
@@ -425,7 +159,7 @@ class BBCooker:
 
             print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
                                         prefstr)
-        
+
 
     def showEnvironment( self ):
         """Show the outer or per-package environment"""
@@ -433,268 +167,190 @@ class BBCooker:
             self.cb = None
             self.bb_cache = bb.cache.init(self)
             try:
-                self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self)
+                self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self.configuration.data)
             except IOError, e:
-                fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e ))
+                bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
             except Exception, e:
-                fatal("%s" % e)
+                bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
         # emit variables and shell functions
         try:
             data.update_data( self.configuration.data )
             data.emit_env(sys.__stdout__, self.configuration.data, True)
         except Exception, e:
-            fatal("%s" % e)
+            bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
         # emit the metadata which isnt valid shell
+        data.expandKeys( self.configuration.data )	
         for e in self.configuration.data.keys():
             if data.getVarFlag( e, 'python', self.configuration.data ):
                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
 
-    def filterProviders(self, providers, item):
+    def generateDotGraph( self, pkgs_to_build, ignore_deps ):
         """
-        Take a list of providers and filter/reorder according to the 
-        environment variables and previous build results
-        """
-        eligible = []
-        preferred_versions = {}
-
-        # Collate providers by PN
-        pkg_pn = {}
-        for p in providers:
-            pn = self.status.pkg_fn[p]
-            if pn not in pkg_pn:
-                pkg_pn[pn] = []
-            pkg_pn[pn].append(p)
-
-        bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
-
-        for pn in pkg_pn.keys():
-            preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
-            eligible.append(preferred_versions[pn][1])
-
-        for p in eligible:
-            if p in self.build_cache_fail:
-                bb.debug(1, "rejecting already-failed %s" % p)
-                eligible.remove(p)
-
-        if len(eligible) == 0:
-            bb.error("no eligible providers for %s" % item)
-            return 0
+        Generate two graphs one for the DEPENDS and RDEPENDS. The current
+        implementation creates crappy graphs ;)
 
-        # look to see if one of them is already staged, or marked as preferred.
-        # if so, bump it to the head of the queue
-        for p in providers:
-            pn = self.status.pkg_fn[p]
-            pv, pr = self.status.pkg_pvpr[p]
-
-            stamp = '%s.do_populate_staging' % self.status.stamp[p]
-            if os.path.exists(stamp):
-                (newvers, fn) = preferred_versions[pn]
-                if not fn in eligible:
-                    # package was made ineligible by already-failed check
-                    continue
-                oldver = "%s-%s" % (pv, pr)
-                newver = '-'.join(newvers)
-                if (newver != oldver):
-                    extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item)
-                else:
-                    extra_chat = "Selecting already-staged %s (%s) to satisfy %s" % (pn, oldver, item)
-                if self.configuration.verbose:
-                    bb.note("%s" % extra_chat)
-                eligible.remove(fn)
-                eligible = [fn] + eligible
-                discriminated = True
-                break
-
-        return eligible
-
-    def buildProvider( self, item , buildAllDeps , build_depends = [] ):
+        pkgs_to_build A list of packages that needs to be built
+        ignore_deps   A list of names where processing of dependencies
+                      should be stopped. e.g. dependencies that get
         """
-        Build something to provide a named build requirement
-        (takes item names from DEPENDS namespace)
-        """
-
-        fn = None
-        discriminated = False
-
-        if not item in self.status.providers:
-            bb.error("Nothing provides dependency %s" % item)
-            bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
-            return 0
 
-        all_p = self.status.providers[item]
-
-        for p in all_p:
-            if p in self.build_cache:
-                bb.debug(1, "already built %s in this run\n" % p)
-                return 1
-
-        eligible = self.filterProviders(all_p, item)
+        def myFilterProvider( providers, item):
+            """
+            Take a list of providers and filter according to environment
+            variables. In contrast to filterProviders we do not discriminate
+            and take PREFERRED_PROVIDER into account.
+            """
+            eligible = []
+            preferred_versions = {}
+
+            # Collate providers by PN
+            pkg_pn = {}
+            for p in providers:
+                pn = self.status.pkg_fn[p]
+                if pn not in pkg_pn:
+                    pkg_pn[pn] = []
+                pkg_pn[pn].append(p)
 
-        if not eligible:
-            return 0
+            bb.msg.debug(1, bb.msg.domain.Provider, "providers for %s are: %s" % (item, pkg_pn.keys()))
 
-        prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
-        if