Initial population

git-svn-id: https://svn.o-hand.com/repos/poky@2 311d38ba-8fff-0310-9ca6-ca027cbcb966

56 files changed, 8930 insertions, 0 deletions

diff --git a/bitbake/AUTHORS b/bitbake/AUTHORS
new file mode 100644
index 0000000000..4129e4c523
--- /dev/null
+++ b/bitbake/AUTHORS
@@ -0,0 +1,5 @@
+Holger Freyther <zecke@handhelds.org>
+Chris Larson <kergoth@handhelds.org>
+Mickey Lauer <mickey@Vanille.de>
+Holger Schurig <holgerschurig@gmx.de>
+Phil Blundell <pb@handhelds.org>
diff --git a/bitbake/ChangeLog b/bitbake/ChangeLog
new file mode 100644
index 0000000000..db503ffb1a
--- /dev/null
+++ b/bitbake/ChangeLog
@@ -0,0 +1,35 @@
+Changes in BitBake 1.3.2:
+	- reintegration of make.py into BitBake
+	- bbread is gone, use bitbake -e
+	- lots of shell updates and bugfixes
+	- Introduction of the .= and =. operator
+	- Sort variables, keys and groups in bitdoc
+	- Fix regression in the handling of BBCOLLECTIONS
+	- Update the bitbake usermanual
+
+Changes in BitBake 1.3.0:
+	- add bitbake interactive shell (bitbake -i)
+	- refactor bitbake utility in OO style
+	- kill default arguments in methods in the bb.data module
+	- kill default arguments in methods in the bb.fetch module
+	- the http/https/ftp fetcher will fail if the to be 
+	  downloaded file was not found in DL_DIR (this is needed
+	  to avoid unpacking the sourceforge mirror page)
+	- Switch to a cow like data instance for persistent and non
+	  persisting mode (called data_smart.py)
+	- Changed the callback of bb.make.collect_bbfiles to carry
+	  additional parameters
+	- Drastically reduced the amount of needed RAM by not holding
+	  each data instance in memory when using a cache/persistent
+	  storage
+
+Changes in BitBake 1.2.1:
+	The 1.2.1 release is meant as a intermediate release to lay the
+	ground for more radical changes. The most notable changes are:
+
+	- Do not hardcode {}, use bb.data.init() instead if you want to
+	  get a instance of a data class
+	- bb.data.init() is a factory and the old bb.data methods are delegates
+	- Do not use deepcopy use bb.data.createCopy() instead.
+	- Removed default arguments in bb.fetch
+
diff --git a/bitbake/MANIFEST b/bitbake/MANIFEST
new file mode 100644
index 0000000000..71e3605a47
--- /dev/null
+++ b/bitbake/MANIFEST
@@ -0,0 +1,27 @@
+AUTHORS
+ChangeLog
+MANIFEST
+setup.py
+bin/bbimage
+bin/bitbake
+lib/bb/__init__.py
+lib/bb/build.py
+lib/bb/data.py
+lib/bb/data_smart.py
+lib/bb/event.py
+lib/bb/fetch.py
+lib/bb/manifest.py
+lib/bb/parse/__init__.py
+lib/bb/parse/parse_py/BBHandler.py
+lib/bb/parse/parse_py/ConfHandler.py
+lib/bb/parse/parse_py/__init__.py
+lib/bb/shell.py
+lib/bb/utils.py
+doc/COPYING.GPL
+doc/COPYING.MIT
+doc/manual/html.css
+doc/manual/Makefile
+doc/manual/usermanual.xml
+contrib/bbdev.sh
+conf/bitbake.conf
+classes/base.bbclass
diff --git a/bitbake/TODO b/bitbake/TODO
new file mode 100644
index 0000000000..511fae4a25
--- /dev/null
+++ b/bitbake/TODO
@@ -0,0 +1,18 @@
+On popular request by popular people a list of tasks to-do:
+
+	-Kill insecure usage of os.system either by properly escaping
+         the strings or a faster replacement not involving /bin/sh
+	-Introduce a -p option to automatically hotshot/profile the
+	 run
+	-Cache dependencies separately and invalidate them when any file
+         changed.
+	-...
+
+
+DONE:
+·       -On generating the inter package deps do not parse each file multiply
+·        times.
+	-We build the lists while parsing the data now
+·       (WAS: Do not generate the world dependency tree, only when someone
+·             requests it.
+
diff --git a/bitbake/bin/bbimage b/bitbake/bin/bbimage
new file mode 100755
index 0000000000..df6caa28ed
--- /dev/null
+++ b/bitbake/bin/bbimage
@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2003  Chris Larson
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+# 
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+# Place, Suite 330, Boston, MA 02111-1307 USA. 
+
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+import bb
+from bb import *
+
+__version__ = 1.0
+type = "jffs2"
+cfg_bb = data.init()
+cfg_oespawn = data.init()
+
+
+def usage():
+    print "Usage: bbimage [options ...]"
+    print "Creates an image for a target device from a root filesystem,"
+    print "obeying configuration parameters from the BitBake"
+    print "configuration files, thereby easing handling of deviceisms."
+    print ""
+    print "  %s\t\t%s" % ("-r [arg], --root [arg]", "root directory (default=${IMAGE_ROOTFS})")
+    print "  %s\t\t%s" % ("-t [arg], --type [arg]", "image type (jffs2[default], cramfs)")
+    print "  %s\t\t%s" % ("-n [arg], --name [arg]", "image name (override IMAGE_NAME variable)")
+    print "  %s\t\t%s" % ("-v, --version", "output version information and exit")
+    sys.exit(0)
+
+def version():
+    print "BitBake Build Tool Core version %s" % bb.__version__
+    print "BBImage version %s" % __version__
+
+def emit_bb(d, base_d = {}):
+    for v in d.keys():
+        if d[v] != base_d[v]:
+            data.emit_var(v, d)
+
+def getopthash(l):
+    h = {}
+    for (opt, val) in l:
+        h[opt] = val
+    return h
+
+import getopt
+try:
+    (opts, args) = getopt.getopt(sys.argv[1:], 'vr:t:e:n:', [ 'version', 'root=', 'type=', 'bbfile=', 'name=' ])
+except getopt.GetoptError:
+    usage()
+
+# handle opts
+opthash = getopthash(opts)
+
+if '--version' in opthash or '-v' in opthash:
+    version()
+    sys.exit(0)
+
+try:
+    cfg_bb = parse.handle(os.path.join('conf', 'bitbake.conf'), cfg_bb)
+except IOError:
+    fatal("Unable to open bitbake.conf")
+
+# sanity check
+if cfg_bb is None:
+    fatal("Unable to open/parse %s" % os.path.join('conf', 'bitbake.conf'))
+    usage(1)
+
+rootfs = None
+extra_files = []
+
+if '--root' in opthash:
+    rootfs = opthash['--root']
+if '-r' in opthash:
+    rootfs = opthash['-r']
+
+if '--type' in opthash:
+    type = opthash['--type']
+if '-t' in opthash:
+    type = opthash['-t']
+
+if '--bbfile' in opthash:
+    extra_files.append(opthash['--bbfile'])
+if '-e' in opthash:
+    extra_files.append(opthash['-e'])
+
+for f in extra_files:
+    try:
+        cfg_bb = parse.handle(f, cfg_bb)
+    except IOError:
+        print "unable to open %s" % f
+
+if not rootfs:
+    rootfs = data.getVar('IMAGE_ROOTFS', cfg_bb, 1)
+
+if not rootfs:
+    bb.fatal("IMAGE_ROOTFS not defined")
+
+data.setVar('IMAGE_ROOTFS', rootfs, cfg_bb)
+
+from copy import copy, deepcopy
+localdata = data.createCopy(cfg_bb)
+
+overrides = data.getVar('OVERRIDES', localdata)
+if not overrides:
+    bb.fatal("OVERRIDES not defined.")
+data.setVar('OVERRIDES', '%s:%s' % (overrides, type), localdata)
+data.update_data(localdata)
+data.setVar('OVERRIDES', overrides, localdata)
+
+if '-n' in opthash:
+    data.setVar('IMAGE_NAME', opthash['-n'], localdata)
+if '--name' in opthash:
+    data.setVar('IMAGE_NAME', opthash['--name'], localdata)
+
+topdir = data.getVar('TOPDIR', localdata, 1) or os.getcwd()
+
+cmd = data.getVar('IMAGE_CMD', localdata, 1)
+if not cmd:
+    bb.fatal("IMAGE_CMD not defined")
+
+outdir = data.getVar('DEPLOY_DIR_IMAGE', localdata, 1)
+if not outdir:
+    bb.fatal('DEPLOY_DIR_IMAGE not defined')
+mkdirhier(outdir)
+
+#depends = data.getVar('IMAGE_DEPENDS', localdata, 1) or ""
+#if depends:
+#       bb.note("Spawning bbmake to satisfy dependencies: %s" % depends)
+#       ret = os.system('bbmake %s' % depends)
+#       if ret != 0:
+#           bb.error("executing bbmake to satisfy dependencies")
+
+bb.note("Executing %s" % cmd)
+data.setVar('image_cmd', cmd, localdata)
+data.setVarFlag('image_cmd', 'func', 1, localdata)
+try:
+    bb.build.exec_func('image_cmd', localdata)
+except bb.build.FuncFailed:
+    sys.exit(1)
+#ret = os.system(cmd)
+#sys.exit(ret)
diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
new file mode 100755
index 0000000000..d327a69ab8
--- /dev/null
+++ b/bitbake/bin/bitbake
@@ -0,0 +1,927 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2003, 2004  Chris Larson
+# Copyright (C) 2003, 2004  Phil Blundell
+# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
+# Copyright (C) 2005        Holger Hans Peter Freyther
+# Copyright (C) 2005        ROAD GmbH
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+# Place, Suite 330, Boston, MA 02111-1307 USA.
+
+import sys, os, getopt, glob, copy, os.path, re, time
+sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+import bb
+from bb import utils, data, parse, debug, event, fatal
+from sets import Set
+import itertools, optparse
+
+parsespin = itertools.cycle( r'|/-\\' )
+bbdebug = 0
+
+__version__ = "1.3.2"
+
+#============================================================================#
+# BBParsingStatus
+#============================================================================#
+class BBParsingStatus:
+    """
+    The initial idea for this status class is to use the data when it is
+    already loaded instead of loading it from various place over and over
+    again.
+    """
+
+    def __init__(self):
+        self.cache_dirty = False
+        self.providers   = {}
+        self.bbfile_priority = {}
+        self.bbfile_config_priorities = []
+        self.ignored_depedencies = None
+        self.possible_world = []
+        self.world_target = Set()
+        self.pkg_pn = {}
+        self.pkg_fn = {}
+        self.pkg_pvpr = {}
+        self.pkg_dp = {}
+        self.pn_provides = {}
+        self.all_depends = Set()
+
+    def handle_bb_data(self, file_name, bb_data, cached):
+        """
+        We will fill the dictionaries with the stuff we
+        need for building the tree more fast
+        """
+        if bb_data == None:
+            return
+
+        if not cached:
+            self.cache_dirty = True
+
+        pn       = bb.data.getVar('PN', bb_data, True)
+        pv       = bb.data.getVar('PV', bb_data, True)
+        pr       = bb.data.getVar('PR', bb_data, True)
+        dp       = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0")
+        provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split())
+        depends  = (bb.data.getVar("DEPENDS", bb_data, True) or "").split()
+
+
+        # build PackageName to FileName lookup table
+        if pn not in self.pkg_pn:
+            self.pkg_pn[pn] = []
+        self.pkg_pn[pn].append(file_name)
+
+        # build FileName to PackageName lookup table
+        self.pkg_fn[file_name] = pn
+        self.pkg_pvpr[file_name] = (pv,pr)
+        self.pkg_dp[file_name] = dp
+
+        # Build forward and reverse provider hashes
+        # Forward: virtual -> [filenames]
+        # Reverse: PN -> [virtuals]
+        if pn not in self.pn_provides:
+            self.pn_provides[pn] = Set()
+        self.pn_provides[pn] |= provides
+
+        for provide in provides:
+            if provide not in self.providers:
+                self.providers[provide] = []
+            self.providers[provide].append(file_name)
+
+        for dep in depends:
+            self.all_depends.add(dep)
+
+        # Collect files we may need for possible world-dep
+        # calculations
+        if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True):
+            self.possible_world.append(file_name)
+
+
+#============================================================================#
+# BBStatistics
+#============================================================================#
+class BBStatistics:
+    """
+    Manage build statistics for one run
+    """
+    def __init__(self ):
+        self.attempt = 0
+        self.success = 0
+        self.fail = 0
+        self.deps = 0
+
+    def show( self ):
+        print "Build statistics:"
+        print "  Attempted builds: %d" % self.attempt
+        if self.fail:
+            print "  Failed builds: %d" % self.fail
+        if self.deps:
+            print "  Dependencies not satisfied: %d" % self.deps
+        if self.fail or self.deps: return 1
+        else: return 0
+
+
+#============================================================================#
+# BBOptions
+#============================================================================#
+class BBConfiguration( object ):
+    """
+    Manages build options and configurations for one run
+    """
+    def __init__( self, options ):
+        for key, val in options.__dict__.items():
+            setattr( self, key, val )
+        self.data = data.init()
+
+#============================================================================#
+# BBCooker
+#============================================================================#
+class BBCooker:
+    """
+    Manages one bitbake build run
+    """
+
+    ParsingStatus = BBParsingStatus     # make it visible from the shell
+    Statistics = BBStatistics           # make it visible from the shell
+
+    def __init__( self ):
+        self.build_cache_fail = []
+        self.build_cache = []
+        self.building_list = []
+        self.build_path = []
+        self.consider_msgs_cache = []
+        self.preferred = {}
+        self.stats = BBStatistics()
+        self.status = None
+
+        self.pkgdata = None
+        self.cache = None
+
+    def tryBuildPackage( self, fn, item, the_data ):
+        """Build one package"""
+        bb.event.fire(bb.event.PkgStarted(item, the_data))
+        try:
+            self.stats.attempt += 1
+            if self.configuration.force:
+                bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data)
+            if not self.configuration.dry_run:
+                bb.build.exec_task('do_%s' % self.configuration.cmd, the_data)
+            bb.event.fire(bb.event.PkgSucceeded(item, the_data))
+            self.build_cache.append(fn)
+            return True
+        except bb.build.FuncFailed:
+            self.stats.fail += 1
+            bb.error("task stack execution failed")
+            bb.event.fire(bb.event.PkgFailed(item, the_data))
+            self.build_cache_fail.append(fn)
+            raise
+        except bb.build.EventException, e:
+            self.stats.fail += 1
+            event = e.args[1]
+            bb.error("%s event exception, aborting" % bb.event.getName(event))
+            bb.event.fire(bb.event.PkgFailed(item, the_data))
+            self.build_cache_fail.append(fn)
+            raise
+
+    def tryBuild( self, fn, virtual ):
+        """Build a provider and its dependencies"""
+        if fn in self.building_list:
+            bb.error("%s depends on itself (eventually)" % fn)
+            bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
+            return False
+
+        the_data = self.pkgdata[fn]
+        item = self.status.pkg_fn[fn]
+
+        self.building_list.append(fn)
+
+        pathstr = "%s (%s)" % (item, virtual)
+        self.build_path.append(pathstr)
+
+        depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
+        if self.configuration.verbose:
+            bb.note("current path: %s" % (" -> ".join(self.build_path)))
+            bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
+
+        try:
+            failed = False
+
+            depcmd = self.configuration.cmd
+            bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data)
+            if bbdepcmd is not None:
+                if bbdepcmd == "":
+                    depcmd = None
+                else:
+                    depcmd = bbdepcmd
+
+            if depcmd:
+                oldcmd = self.configuration.cmd
+                self.configuration.cmd = depcmd
+
+            for dependency in depends_list:
+                if dependency in self.status.ignored_dependencies:
+                    continue
+                if not depcmd:
+                    continue
+                if self.buildProvider( dependency ) == 0:
+                    bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
+                    failed = True
+                    if self.configuration.abort:
+                        break
+
+            if depcmd:
+                self.configuration.cmd = oldcmd
+
+            if failed:
+                self.stats.deps += 1
+                return False
+
+            if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
+                self.build_cache.append(fn)
+                return True
+
+            return self.tryBuildPackage( fn, item, the_data )
+
+        finally:
+            self.building_list.remove(fn)
+            self.build_path.remove(pathstr)
+
+    def findBestProvider( self, pn, pkg_pn = None):
+        """
+        If there is a PREFERRED_VERSION, find the highest-priority bbfile
+        providing that version.  If not, find the latest version provided by
+        an bbfile in the highest-priority set.
+        """
+        if not pkg_pn:
+            pkg_pn = self.status.pkg_pn
+
+        files = pkg_pn[pn]
+        priorities = {}
+        for f in files:
+            priority = self.status.bbfile_priority[f]
+            if priority not in priorities:
+                priorities[priority] = []
+            priorities[priority].append(f)
+        p_list = priorities.keys()
+        p_list.sort(lambda a, b: a - b)
+        tmp_pn = []
+        for p in p_list:
+            tmp_pn = [priorities[p]] + tmp_pn
+
+        preferred_file = None
+
+        preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, self.configuration.data, 1)
+        if preferred_v:
+            m = re.match('(.*)_(.*)', preferred_v)
+            if m:
+                preferred_v = m.group(1)
+                preferred_r = m.group(2)
+            else:
+                preferred_r = None
+
+            for file_set in tmp_pn:
+                for f in file_set:
+                    pv,pr = self.status.pkg_pvpr[f]
+                    if preferred_v == pv and (preferred_r == pr or preferred_r == None):
+                        preferred_file = f
+                        preferred_ver = (pv, pr)
+                        break
+                if preferred_file:
+                    break;
+            if preferred_r:
+                pv_str = '%s-%s' % (preferred_v, preferred_r)
+            else:
+                pv_str = preferred_v
+            if preferred_file is None:
+                bb.note("preferred version %s of %s not available" % (pv_str, pn))
+            else:
+                bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
+
+        # get highest priority file set
+        files = tmp_pn[0]
+        latest = None
+        latest_p = 0
+        latest_f = None
+        for file_name in files:
+            pv,pr = self.status.pkg_pvpr[file_name]
+            dp = self.status.pkg_dp[file_name]
+
+            if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
+                latest = (pv, pr)
+                latest_f = file_name
+                latest_p = dp
+        if preferred_file is None:
+            preferred_file = latest_f
+            preferred_ver = latest
+
+        return (latest,latest_f,preferred_ver, preferred_file)
+
+    def showVersions( self ):
+        pkg_pn = self.status.pkg_pn
+        preferred_versions = {}
+        latest_versions = {}
+
+        # Sort by priority
+        for pn in pkg_pn.keys():
+            (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
+            preferred_versions[pn] = (pref_ver, pref_file)
+            latest_versions[pn] = (last_ver, last_file)
+
+        pkg_list = pkg_pn.keys()
+        pkg_list.sort()
+
+        for p in pkg_list:
+            pref = preferred_versions[p]
+            latest = latest_versions[p]
+
+            if pref != latest:
+                prefstr = pref[0][0] + "-" + pref[0][1]
+            else:
+                prefstr = ""
+
+            print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
+                                        prefstr)
+
+    def showEnvironment( self ):
+        """Show the outer or per-package environment"""
+        if self.configuration.buildfile:
+            try:
+                self.configuration.data, fromCache = self.load_bbfile( self.configuration.buildfile )
+            except IOError, e:
+                fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e ))
+            except Exception, e:
+                fatal("%s" % e)
+        # emit variables and shell functions
+        try:
+            data.update_data( self.configuration.data )
+            data.emit_env(sys.__stdout__, self.configuration.data, True)
+        except Exception, e:
+            fatal("%s" % e)
+        # emit the metadata which isnt valid shell
+        for e in self.configuration.data.keys():
+            if data.getVarFlag( e, 'python', self.configuration.data ):
+                sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
+
+    def buildProvider( self, item ):
+        fn = None
+
+        discriminated = False
+
+        if item not in self.status.providers:
+            bb.error("Nothing provides %s" % item)
+            return 0
+
+        all_p = self.status.providers[item]
+
+        for p in all_p:
+            if p in self.build_cache:
+                bb.debug(1, "already