13 files changed, 6413 insertions, 0 deletions
diff --git a/scripts/lib/mic/utils/BmapCreate.py b/scripts/lib/mic/utils/BmapCreate.py
new file mode 100644
index 0000000000..65b19a5f46
--- /dev/null
+++ b/scripts/lib/mic/utils/BmapCreate.py
@@ -0,0 +1,298 @@
+""" This module implements the block map (bmap) creation functionality and
+provides the corresponding API in form of the 'BmapCreate' class.
+
+The idea is that while images files may generally be very large (e.g., 4GiB),
+they may nevertheless contain only little real data, e.g., 512MiB. This data
+are files, directories, file-system meta-data, partition table, etc. When
+copying the image to the target device, you do not have to copy all the 4GiB of
+data, you can copy only 512MiB of it, which is 4 times less, so copying should
+presumably be 4 times faster.
+
+The block map file is an XML file which contains a list of blocks which have to
+be copied to the target device. The other blocks are not used and there is no
+need to copy them. The XML file also contains some additional information like
+block size, image size, count of mapped blocks, etc. There are also many
+commentaries, so it is human-readable.
+
+The image has to be a sparse file. Generally, this means that when you generate
+this image file, you should start with a huge sparse file which contains a
+single hole spanning the entire file. Then you should partition it, write all
+the data (probably by means of loop-back mounting the image or parts of it),
+etc. The end result should be a sparse file where mapped areas represent useful
+parts of the image and holes represent useless parts of the image, which do not
+have to be copied when copying the image to the target device.
+
+This module uses the FIBMAP ioctl to detect holes. """
+
+# Disable the following pylint recommendations:
+#   *  Too many instance attributes - R0902
+#   *  Too few public methods - R0903
+# pylint: disable=R0902,R0903
+
+import hashlib
+from mic.utils.misc import human_size
+from mic.utils import Fiemap
+
+# The bmap format version we generate
+SUPPORTED_BMAP_VERSION = "1.3"
+
+_BMAP_START_TEMPLATE = \
+"""<?xml version="1.0" ?>
+<!-- This file contains the block map for an image file, which is basically
+     a list of useful (mapped) block numbers in the image file. In other words,
+     it lists only those blocks which contain data (boot sector, partition
+     table, file-system metadata, files, directories, extents, etc). These
+     blocks have to be copied to the target device. The other blocks do not
+     contain any useful data and do not have to be copied to the target
+     device.
+
+     The block map an optimization which allows to copy or flash the image to
+     the image quicker than copying of flashing the entire image. This is
+     because with bmap less data is copied: <MappedBlocksCount> blocks instead
+     of <BlocksCount> blocks.
+
+     Besides the machine-readable data, this file contains useful commentaries
+     which contain human-readable information like image size, percentage of
+     mapped data, etc.
+
+     The 'version' attribute is the block map file format version in the
+     'major.minor' format. The version major number is increased whenever an
+     incompatible block map format change is made. The minor number changes
+     in case of minor backward-compatible changes. -->
+
+<bmap version="%s">
+    <!-- Image size in bytes: %s -->
+    <ImageSize> %u </ImageSize>
+
+    <!-- Size of a block in bytes -->
+    <BlockSize> %u </BlockSize>
+
+    <!-- Count of blocks in the image file -->
+    <BlocksCount> %u </BlocksCount>
+
+"""
+
+class Error(Exception):
+    """ A class for exceptions generated by this module. We currently support
+    only one type of exceptions, and we basically throw human-readable problem
+    description in case of errors. """
+    pass
+
+class BmapCreate:
+    """ This class implements the bmap creation functionality. To generate a
+    bmap for an image (which is supposedly a sparse file), you should first
+    create an instance of 'BmapCreate' and provide:
+
+    * full path or a file-like object of the image to create bmap for
+    * full path or a file object to use for writing the results to
+
+    Then you should invoke the 'generate()' method of this class. It will use
+    the FIEMAP ioctl to generate the bmap. """
+
+    def _open_image_file(self):
+        """ Open the image file. """
+
+        try:
+            self._f_image = open(self._image_path, 'rb')
+        except IOError as err:
+            raise Error("cannot open image file '%s': %s" \
+                        % (self._image_path, err))
+
+        self._f_image_needs_close = True
+
+    def _open_bmap_file(self):
+        """ Open the bmap file. """
+
+        try:
+            self._f_bmap = open(self._bmap_path, 'w+')
+        except IOError as err:
+            raise Error("cannot open bmap file '%s': %s" \
+                        % (self._bmap_path, err))
+
+        self._f_bmap_needs_close = True
+
+    def __init__(self, image, bmap):
+        """ Initialize a class instance:
+        * image - full path or a file-like object of the image to create bmap
+                  for
+        * bmap  - full path or a file object to use for writing the resulting
+                  bmap to """
+
+        self.image_size = None
+        self.image_size_human = None
+        self.block_size = None
+        self.blocks_cnt = None
+        self.mapped_cnt = None
+        self.mapped_size = None
+        self.mapped_size_human = None
+        self.mapped_percent = None
+
+        self._mapped_count_pos1 = None
+        self._mapped_count_pos2 = None
+        self._sha1_pos = None
+
+        self._f_image_needs_close = False
+        self._f_bmap_needs_close = False
+
+        if hasattr(image, "read"):
+            self._f_image = image
+            self._image_path = image.name
+        else:
+            self._image_path = image
+            self._open_image_file()
+
+        if hasattr(bmap, "read"):
+            self._f_bmap = bmap
+            self._bmap_path = bmap.name
+        else:
+            self._bmap_path = bmap
+            self._open_bmap_file()
+
+        self.fiemap = Fiemap.Fiemap(self._f_image)
+
+        self.image_size = self.fiemap.image_size
+        self.image_size_human = human_size(self.image_size)
+        if self.image_size == 0:
+            raise Error("cannot generate bmap for zero-sized image file '%s'" \
+                        % self._image_path)
+
+        self.block_size = self.fiemap.block_size
+        self.blocks_cnt = self.fiemap.blocks_cnt
+
+    def _bmap_file_start(self):
+        """ A helper function which generates the starting contents of the
+        block map file: the header comment, image size, block size, etc. """
+
+        # We do not know the amount of mapped blocks at the moment, so just put
+        # whitespaces instead of real numbers. Assume the longest possible
+        # numbers.
+        mapped_count = ' ' * len(str(self.image_size))
+        mapped_size_human = ' ' * len(self.image_size_human)
+
+        xml = _BMAP_START_TEMPLATE \
+               % (SUPPORTED_BMAP_VERSION, self.image_size_human,
+                  self.image_size, self.block_size, self.blocks_cnt)
+        xml += "    <!-- Count of mapped blocks: "
+
+        self._f_bmap.write(xml)
+        self._mapped_count_pos1 = self._f_bmap.tell()
+
+        # Just put white-spaces instead of real information about mapped blocks
+        xml  = "%s or %.1f    -->\n" % (mapped_size_human, 100.0)
+        xml += "    <MappedBlocksCount> "
+
+        self._f_bmap.write(xml)
+        self._mapped_count_pos2 = self._f_bmap.tell()
+
+        xml  = "%s </MappedBlocksCount>\n\n" % mapped_count
+
+        # pylint: disable=C0301
+        xml += "    <!-- The checksum of this bmap file. When it is calculated, the value of\n"
+        xml += "         the SHA1 checksum has be zeoro (40 ASCII \"0\" symbols). -->\n"
+        xml += "    <BmapFileSHA1> "
+
+        self._f_bmap.write(xml)
+        self._sha1_pos = self._f_bmap.tell()
+
+        xml = "0" * 40 + " </BmapFileSHA1>\n\n"
+        xml += "    <!-- The block map which consists of elements which may either be a\n"
+        xml += "         range of blocks or a single block. The 'sha1' attribute (if present)\n"
+        xml += "         is the SHA1 checksum of this blocks range. -->\n"
+        xml += "    <BlockMap>\n"
+        # pylint: enable=C0301
+
+        self._f_bmap.write(xml)
+
+    def _bmap_file_end(self):
+        """ A helper function which generates the final parts of the block map
+        file: the ending tags and the information about the amount of mapped
+        blocks. """
+
+        xml =  "    </BlockMap>\n"
+        xml += "</bmap>\n"
+
+        self._f_bmap.write(xml)
+
+        self._f_bmap.seek(self._mapped_count_pos1)
+        self._f_bmap.write("%s or %.1f%%" % \
+                           (self.mapped_size_human, self.mapped_percent))
+
+        self._f_bmap.seek(self._mapped_count_pos2)
+        self._f_bmap.write("%u" % self.mapped_cnt)
+
+        self._f_bmap.seek(0)
+        sha1 = hashlib.sha1(self._f_bmap.read()).hexdigest()
+        self._f_bmap.seek(self._sha1_pos)
+        self._f_bmap.write("%s" % sha1)
+
+    def _calculate_sha1(self, first, last):
+        """ A helper function which calculates SHA1 checksum for the range of
+        blocks of the image file: from block 'first' to block 'last'. """
+
+        start = first * self.block_size
+        end = (last + 1) * self.block_size
+
+        self._f_image.seek(start)
+        hash_obj = hashlib.new("sha1")
+
+        chunk_size = 1024*1024
+        to_read = end - start
+        read = 0
+
+        while read < to_read:
+            if read + chunk_size > to_read:
+                chunk_size = to_read - read
+            chunk = self._f_image.read(chunk_size)
+            hash_obj.update(chunk)
+            read += chunk_size
+
+        return hash_obj.hexdigest()
+
+    def generate(self, include_checksums = True):
+        """ Generate bmap for the image file. If 'include_checksums' is 'True',
+        also generate SHA1 checksums for block ranges. """
+
+        # Save image file position in order to restore it at the end
+        image_pos = self._f_image.tell()
+
+        self._bmap_file_start()
+
+        # Generate the block map and write it to the XML block map
+        # file as we go.
+        self.mapped_cnt = 0
+        for first, last in self.fiemap.get_mapped_ranges(0, self.blocks_cnt):
+            self.mapped_cnt += last - first + 1
+            if include_checksums:
+                sha1 = self._calculate_sha1(first, last)
+                sha1 = " sha1=\"%s\"" % sha1
+            else:
+                sha1 = ""
+
+            if first != last:
+                self._f_bmap.write("        <Range%s> %s-%s </Range>\n" \
+                                   % (sha1, first, last))
+            else:
+                self._f_bmap.write("        <Range%s> %s </Range>\n" \
+                                   % (sha1, first))
+
+        self.mapped_size = self.mapped_cnt * self.block_size
+        self.mapped_size_human = human_size(self.mapped_size)
+        self.mapped_percent = (self.mapped_cnt * 100.0) /  self.blocks_cnt
+
+        self._bmap_file_end()
+
+        try:
+            self._f_bmap.flush()
+        except IOError as err:
+            raise Error("cannot flush the bmap file '%s': %s" \
+                        % (self._bmap_path, err))
+
+        self._f_image.seek(image_pos)
+
+    def __del__(self):
+        """ The class destructor which closes the opened files. """
+
+        if self._f_image_needs_close:
+            self._f_image.close()
+        if self._f_bmap_needs_close:
+            self._f_bmap.close()
diff --git a/scripts/lib/mic/utils/Fiemap.py b/scripts/lib/mic/utils/Fiemap.py
new file mode 100644
index 0000000000..f2db6ff0b8
--- /dev/null
+++ b/scripts/lib/mic/utils/Fiemap.py
@@ -0,0 +1,252 @@
+""" This module implements python API for the FIEMAP ioctl. The FIEMAP ioctl
+allows to find holes and mapped areas in a file. """
+
+# Note, a lot of code in this module is not very readable, because it deals
+# with the rather complex FIEMAP ioctl. To understand the code, you need to
+# know the FIEMAP interface, which is documented in the
+# Documentation/filesystems/fiemap.txt file in the Linux kernel sources.
+
+# Disable the following pylint recommendations:
+#   * Too many instance attributes (R0902)
+# pylint: disable=R0902
+
+import os
+import struct
+import array
+import fcntl
+from mic.utils.misc import get_block_size
+
+# Format string for 'struct fiemap'
+_FIEMAP_FORMAT = "=QQLLLL"
+# sizeof(struct fiemap)
+_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
+# Format string for 'struct fiemap_extent'
+_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
+# sizeof(struct fiemap_extent)
+_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
+# The FIEMAP ioctl number
+_FIEMAP_IOCTL = 0xC020660B
+
+# Minimum buffer which is required for 'class Fiemap' to operate
+MIN_BUFFER_SIZE = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE
+# The default buffer size for 'class Fiemap'
+DEFAULT_BUFFER_SIZE = 256 * 1024
+
+class Error(Exception):
+    """ A class for exceptions generated by this module. We currently support
+    only one type of exceptions, and we basically throw human-readable problem
+    description in case of errors. """
+    pass
+
+class Fiemap:
+    """ This class provides API to the FIEMAP ioctl. Namely, it allows to
+    iterate over all mapped blocks and over all holes. """
+
+    def _open_image_file(self):
+        """ Open the image file. """
+
+        try:
+            self._f_image = open(self._image_path, 'rb')
+        except IOError as err:
+            raise Error("cannot open image file '%s': %s" \
+                        % (self._image_path, err))
+
+        self._f_image_needs_close = True
+
+    def __init__(self, image, buf_size = DEFAULT_BUFFER_SIZE):
+        """ Initialize a class instance. The 'image' argument is full path to
+        the file to operate on, or a file object to operate on.
+
+        The 'buf_size' argument is the size of the buffer for 'struct
+        fiemap_extent' elements which will be used when invoking the FIEMAP
+        ioctl. The larger is the buffer, the less times the FIEMAP ioctl will
+        be invoked. """
+
+        self._f_image_needs_close = False
+
+        if hasattr(image, "fileno"):
+            self._f_image = image
+            self._image_path = image.name
+        else:
+            self._image_path = image
+            self._open_image_file()
+
+        # Validate 'buf_size'
+        if buf_size < MIN_BUFFER_SIZE:
+            raise Error("too small buffer (%d bytes), minimum is %d bytes" \
+                    % (buf_size, MIN_BUFFER_SIZE))
+
+        # How many 'struct fiemap_extent' elements fit the buffer
+        buf_size -= _FIEMAP_SIZE
+        self._fiemap_extent_cnt = buf_size / _FIEMAP_EXTENT_SIZE
+        self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE
+        self._buf_size += _FIEMAP_SIZE
+
+        # Allocate a mutable buffer for the FIEMAP ioctl
+        self._buf = array.array('B', [0] * self._buf_size)
+
+        self.image_size = os.fstat(self._f_image.fileno()).st_size
+
+        try:
+            self.block_size = get_block_size(self._f_image)
+        except IOError as err:
+            raise Error("cannot get block size for '%s': %s" \
+                        % (self._image_path, err))
+
+        self.blocks_cnt = self.image_size + self.block_size - 1
+        self.blocks_cnt /= self.block_size
+
+        # Synchronize the image file to make sure FIEMAP returns correct values
+        try:
+            self._f_image.flush()
+        except IOError as err:
+            raise Error("cannot flush image file '%s': %s" \
+                        % (self._image_path, err))
+        try:
+            os.fsync(self._f_image.fileno()),
+        except OSError as err:
+            raise Error("cannot synchronize image file '%s': %s " \
+                        % (self._image_path, err.strerror))
+
+        # Check if the FIEMAP ioctl is supported
+        self.block_is_mapped(0)
+
+    def __del__(self):
+        """ The class destructor which closes the opened files. """
+
+        if self._f_image_needs_close:
+            self._f_image.close()
+
+    def _invoke_fiemap(self, block, count):
+        """ Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
+        block number 'block'.
+
+        The full result of the operation is stored in 'self._buf' on exit.
+        Returns the unpacked 'struct fiemap' data structure in form of a python
+        list (just like 'struct.upack()'). """
+
+        if block < 0 or block >= self.blocks_cnt:
+            raise Error("bad block number %d, should be within [0, %d]" \
+                        % (block, self.blocks_cnt))
+
+        # Initialize the 'struct fiemap' part of the buffer
+        struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size,
+                         count * self.block_size, 0, 0,
+                         self._fiemap_extent_cnt, 0)
+
+        try:
+            fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1)
+        except IOError as err:
+            error_msg = "the FIEMAP ioctl failed for '%s': %s" \
+                        % (self._image_path, err)
+            if err.errno == os.errno.EPERM or err.errno == os.errno.EACCES:
+                # The FIEMAP ioctl was added in kernel version 2.6.28 in 2008
+                error_msg += " (looks like your kernel does not support FIEMAP)"
+
+            raise Error(error_msg)
+
+        return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE])
+
+    def block_is_mapped(self, block):
+        """ This function returns 'True' if block number 'block' of the image
+        file is mapped and 'False' otherwise. """
+
+        struct_fiemap = self._invoke_fiemap(block, 1)
+
+        # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field.
+        # If it contains zero, the block is not mapped, otherwise it is
+        # mapped.
+        return bool(struct_fiemap[3])
+
+    def block_is_unmapped(self, block):
+        """ This function returns 'True' if block number 'block' of the image
+        file is not mapped (hole) and 'False' otherwise. """
+
+        return not self.block_is_mapped(block)
+
+    def _unpack_fiemap_extent(self, index):
+        """ Unpack a 'struct fiemap_extent' structure object number 'index'
+        from the internal 'self._buf' buffer. """
+
+        offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index
+        return struct.unpack(_FIEMAP_EXTENT_FORMAT,
+                             self._buf[offset : offset + _FIEMAP_EXTENT_SIZE])
+
+    def _do_get_mapped_ranges(self, start, count):
+        """ Implements most the functionality for the  'get_mapped_ranges()'
+        generator: invokes the FIEMAP ioctl, walks through the mapped
+        extents and yields mapped block ranges. However, the ranges may be
+        consecutive (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()'
+        simply merges them. """
+
+        block = start
+        while block < start + count:
+            struct_fiemap = self._invoke_fiemap(block, count)
+
+            mapped_extents = struct_fiemap[3]
+            if mapped_extents == 0:
+                # No more mapped blocks
+                return
+
+            extent = 0
+            while extent < mapped_extents:
+                fiemap_extent = self._unpack_fiemap_extent(extent)
+
+                # Start of the extent
+                extent_start = fiemap_extent[0]
+                # Starting block number of the extent
+                extent_block = extent_start / self.block_size
+                # Length of the extent
+                extent_len = fiemap_extent[2]
+                # Count of blocks in the extent
+                extent_count = extent_len / self.block_size
+
+                # Extent length and offset have to be block-aligned
+                assert extent_start % self.block_size == 0
+                assert extent_len % self.block_size == 0
+
+                if extent_block > start + count - 1:
+                    return
+
+                first = max(extent_block, block)
+                last = min(extent_block + extent_count, start + count) - 1
+                yield (first, last)
+
+                extent += 1
+
+            block = extent_block + extent_count
+
+    def get_mapped_ranges(self, start, count):
+        """ A generator which yields ranges of mapped blocks in the file. The
+        ranges are tuples of 2 elements: [first, last], where 'first' is the
+        first mapped block and 'last' is the last mapped block.
+
+        The ranges are yielded for the area of the file of size 'count' blocks,
+        starting from block 'start'. """
+
+        iterator = self._do_get_mapped_ranges(start, count)
+
+        first_prev, last_prev = iterator.next()
+
+        for first, last in iterator:
+            if last_prev == first - 1:
+                last_prev = last
+            else:
+                yield (first_prev, last_prev)
+                first_prev, last_prev = first, last
+
+        yield (first_prev, last_prev)
+
+    def get_unmapped_ranges(self, start, count):
+        """ Just like 'get_mapped_ranges()', but yields unmapped block ranges
+        instead (holes). """
+
+        hole_first = start
+        for first, last in self._do_get_mapped_ranges(start, count):
+            if first > hole_first:
+                yield (hole_first, first - 1)
+
+            hole_first = last + 1
+
+        if hole_first < start + count:
+            yield (hole_first, start + count - 1)
diff --git a/scripts/lib/mic/utils/__init__.py b/scripts/lib/mic/utils/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/scripts/lib/mic/utils/__init__.py
diff --git a/scripts/lib/mic/utils/cmdln.py b/scripts/lib/mic/utils/cmdln.py
new file mode 100644
index 0000000000..b099473ee4
--- /dev/null
+++ b/scripts/lib/mic/utils/cmdln.py
@@ -0,0 +1,1586 @@
+#!/usr/bin/env python
+# Copyright (c) 2002-2007 ActiveState Software Inc.
+# License: MIT (see LICENSE.txt for license details)
+# Author:  Trent Mick
+# Home:    http://trentm.com/projects/cmdln/
+
+"""An improvement on Python's standard cmd.py module.
+
+As with cmd.py, this module provides "a simple framework for writing
+line-oriented command intepreters."  This module provides a 'RawCmdln'
+class that fixes some design flaws in cmd.Cmd, making it more scalable
+and nicer to use for good 'cvs'- or 'svn'-style command line interfaces
+or simple shells.  And it provides a 'Cmdln' class that add
+optparse-based option processing. Basically you use it like this:
+
+    import cmdln
+
+    class MySVN(cmdln.Cmdln):
+        name = "svn"
+
+        @cmdln.alias('stat', 'st')
+        @cmdln.option('-v', '--verbose', action='store_true'
+                      help='print verbose information')
+        def do_status(self, subcmd, opts, *paths):
+            print "handle 'svn status' command"
+
+        #...
+
+    if __name__ == "__main__":
+        shell = MySVN()
+        retval = shell.main()
+        sys.exit(retval)
+
+See the README.txt or <http://trentm.com/projects/cmdln/> for more
+details.
+"""
+
+__version_info__ = (1, 1, 2)
+__version__ = '.'.join(map(str, __version_info__))
+
+import os
+import sys
+import re
+import cmd
+import optparse
+from pprint import pprint
+import sys
+
+
+
+
+#---- globals
+
+LOOP_ALWAYS, LOOP_NEVER, LOOP_IF_EMPTY = range(3)
+
+# An unspecified optional argument when None is a meaningful value.
+_NOT_SPECIFIED = ("Not", "Specified")
+
+# Pattern to match a TypeError message from a call that
+# failed because of incorrect number of arguments (see
+# Python/getargs.c).
+_INCORRECT_NUM_ARGS_RE = re.compile(
+    r"(takes [\w ]+ )(\d+)( arguments? \()(\d+)( given\))")
+
+
+
+#---- exceptions
+
+class CmdlnError(Exception):
+    """A cmdln.py usage error."""
+    def __init__(self, msg):
+        self.msg = msg
+    def __str__(self):
+        return self.msg
+
+class CmdlnUserError(Exception):
+    """An error by a user of a cmdln-based tool/shell."""
+    pass
+
+
+
+#---- public methods and classes
+
+def alias(*aliases):
+    """Decorator to add aliases for Cmdln.do_* command handlers.
+    
+    Example:
+        class MyShell(cmdln.Cmdln):
+            @cmdln.alias("!", "sh")
+            def do_shell(self, argv):
+                #...implement 'shell' command
+    """
+    def decorate(f):
+        if not hasattr(f, "aliases"):
+            f.aliases = []
+        f.aliases += aliases
+        return f
+    return decorate
+
+
+class RawCmdln(cmd.Cmd):
+    """An improved (on cmd.Cmd) framework for building multi-subcommand
+    scripts (think "svn" & "cvs") and simple shells (think "pdb" and
+    "gdb").
+
+    A simple example:
+
+        import cmdln
+
+        class MySVN(cmdln.RawCmdln):
+            name = "svn"
+
+            @cmdln.aliases('stat', 'st')
+            def do_status(self, argv):
+                print "handle 'svn status' command"
+
+        if __name__ == "__main__":
+            shell = MySVN()
+            retval = shell.main()
+            sys.exit(retval)
+
+    See <http://trentm.com/projects/cmdln> for more information.
+    """
+    name = None      # if unset, defaults basename(sys.argv[0])
+    prompt = None    # if unset, defaults to self.name+"> "
+    version = None   # if set, default top-level options include --version
+
+    # Default messages for some 'help' command error cases.
+    # They are interpolated with one arg: the command.
+    nohelp = "no help on '%s'"
+    unknowncmd = "unknown command: '%s'"
+
+    helpindent = '' # string with which to indent help output
+
+    def __init__(self, completekey='tab', 
+                 stdin=None, stdout=None, stderr=None):
+        """Cmdln(completekey='tab', stdin=None, stdout=None, stderr=None)
+
+        The optional argument 'completekey' is the readline name of a
+        completion key; it defaults to the Tab key. If completekey is
+        not None and the readline module is available, command completion
+        is done automatically.
+        
+        The optional arguments 'stdin', 'stdout' and 'stderr' specify
+        alternate input, output and error output file objects; if not
+        specified, sys.* are used.
+        
+        If 'stdout' but not 'stderr' is specified, stdout is used for
+        error output. This is to provide least surprise for users used
+        to only the 'stdin' and 'stdout' options with cmd.Cmd.
+        """
+        import sys
+        if self.name is None:
+            self.name = os.path.basename(sys.argv[0])
+        if self.prompt is None:
+            self.prompt = self.name+"> "
+        self._name_str = self._str(self.name)
+        self._prompt_str = self._str(self.prompt)
+        if stdin is not None:
+            self.stdin = stdin
+        else:
+            self.stdin = sys.stdin
+        if stdout is not None:
+            self.stdout = stdout
+        else:
+            self.stdout = sys.stdout
+        if stderr is not None:
+            self.stderr = stderr
+        elif stdout is not None:
+            self.stderr = stdout
+        else:
+            self.stderr = sys.stderr
+        self.cmdqueue = []
+        self.completekey = completekey
+        self.cmdlooping = False
+
+    def get_optparser(self):
+        """Hook for subclasses to set the option parser for the
+        top-level command/shell.
+
+        This option parser is used retrieved and used by `.main()' to
+        handle top-level options.
+
+        The default implements a single '-h|--help' option. Sub-classes
+        can return None to have no options at the top-level. Typically
+        an instance of CmdlnOptionParser should be returned.
+        """
+        version = (self.version is not None 
+                    and "%s %s" % (self._name_str, self.version)
+                    or None)
+        return CmdlnOptionParser(self, version=version)
+
+    def postoptparse(self):
+        """Hook method executed just after `.main()' parses top-level
+        options.
+
+        When called `self.options' holds the results of the option parse.
+        """
+        pass
+
+    def main(self, argv=None, loop=LOOP_NEVER):
+        """A possible mainline handler for a script, like so:
+
+            import cmdln
+            class MyCmd(cmdln.Cmdln):
+                name = "mycmd"
+                ...
+            
+            if __name__ == "__main__":
+                MyCmd().main()
+
+        By default this will use sys.argv to issue a single command to
+        'MyCmd', then exit. The 'loop' argument can be use to control
+        interactive shell behaviour.
+        
+        Arguments:
+            "argv" (optional, default sys.argv) is the command to run.
+                It must be a sequence, where the first element is the
+                command name and subsequent elements the args for that
+                command.
+            "loop" (optional, default LOOP_NEVER) is a constant
+                indicating if a command loop should be started (i.e. an
+                interactive shell). Valid values (constants on this module):
+                    LOOP_ALWAYS     start loop and run "argv", if any
+                    LOOP_NEVER      run "argv" (or .emptyline()) and exit
+                    LOOP_IF_EMPTY   run "argv", if given, and exit;
+                                    otherwise, start loop
+        """
+        if argv is None:
+            import sys
+            argv = sys.argv
+        else:
+            argv = argv[:] # don't modify caller's list
+
+        self.optparser = self.get_optparser()
+        if self.optparser: # i.e. optparser=None means don't process for opts
+            try:
+                self.options, args = self.optparser.parse_args(argv[1:])
+            except CmdlnUserError, ex:
+                msg = "%s: %s\nTry '%s help' for info.\n"\
+                      % (self.name, ex, self.name)
+                self.stderr.write(self._str(msg))
+                self.stderr.flush()
+                return 1
+            except StopOptionProcessing, ex:
+                return 0
+        else:
+            self.options, args = None, argv[1:]
+        self.postoptparse()
+
+        if loop == LOOP_ALWAYS:
+            if args:
+                self.cmdqueue.append(args)
+            return self.cmdloop()
+        elif loop == LOOP_NEVER:
+            if args:
+                return self.cmd(args)
+            else:
+                return self.emptyline()
+        elif loop == LOOP_IF_EMPTY:
+            if args:
+                return self.cmd(args)
+            else:
+                return self.cmdloop()
+
+    def cmd(self, argv):
+        """Run one command and exit.
+        
+            "argv" is the arglist for the command to run. argv[0] is the
+                command to run. If argv is an empty list then the
+                'emptyline' handler is run.
+
+        Returns the return value from the command handler.
+        """
+        assert isinstance(argv, (list, tuple)), \
+                "'argv' is not a sequence: %r" % argv
+        retval = None
+        try:
+            argv = self.precmd(argv)
+            retval = self.onecmd(argv)
+            self.postcmd(argv)
+        except:
+            if not self.cmdexc(argv):
+                raise
+            retval = 1
+        return retval
+
+    def _str(self, s):
+        """Safely convert the given str/unicode to a string for printing."""
+        try:
+            return str(s)
+        except UnicodeError:
+            #XXX What is the proper encoding to use here? 'utf-8' seems
+            #    to work better than "getdefaultencoding" (usually
+            #    'ascii'), on OS X at least.
+            #import sys
+            #return s.encode(sys.getdefaultencoding(), "replace")
+            return s.encode("utf-8", "replace")
+
+    def cmdloop(self, intro=None):
+        """Repeatedly issue a prompt, accept input, parse into an argv, and
+        dispatch (via .precmd(), .onecmd() and .postcmd()), passing them
+        the argv. In other words, start a shell.
+        
+            "intro" (optional) is a introductory message to print when
+                starting the command loop. This overrides the class
+                "intro" attribute, if any.
+        """
+        self.cmdlooping = True
+        self.preloop()
+        if self.use_rawinput and self.completekey:
+            try:
+                import readline
+                self.old_completer = readline.get_completer()
+                readline.set_completer(self.complete)