diff options
Diffstat (limited to 'scripts/lib/mic/utils')
| -rw-r--r-- | scripts/lib/mic/utils/BmapCreate.py | 298 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/Fiemap.py | 252 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/__init__.py | 0 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/cmdln.py | 1586 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/errors.py | 71 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/fs_related.py | 1060 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/gpt_parser.py | 331 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/grabber.py | 97 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/misc.py | 1065 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/oe/__init__.py | 22 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/oe/misc.py | 174 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/partitionedfs.py | 782 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/proxy.py | 183 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/rpmmisc.py | 600 | ||||
| -rw-r--r-- | scripts/lib/mic/utils/runner.py | 109 |
15 files changed, 0 insertions, 6630 deletions
diff --git a/scripts/lib/mic/utils/BmapCreate.py b/scripts/lib/mic/utils/BmapCreate.py deleted file mode 100644 index 65b19a5f46..0000000000 --- a/scripts/lib/mic/utils/BmapCreate.py +++ /dev/null @@ -1,298 +0,0 @@ -""" This module implements the block map (bmap) creation functionality and -provides the corresponding API in form of the 'BmapCreate' class. - -The idea is that while images files may generally be very large (e.g., 4GiB), -they may nevertheless contain only little real data, e.g., 512MiB. This data -are files, directories, file-system meta-data, partition table, etc. When -copying the image to the target device, you do not have to copy all the 4GiB of -data, you can copy only 512MiB of it, which is 4 times less, so copying should -presumably be 4 times faster. - -The block map file is an XML file which contains a list of blocks which have to -be copied to the target device. The other blocks are not used and there is no -need to copy them. The XML file also contains some additional information like -block size, image size, count of mapped blocks, etc. There are also many -commentaries, so it is human-readable. - -The image has to be a sparse file. Generally, this means that when you generate -this image file, you should start with a huge sparse file which contains a -single hole spanning the entire file. Then you should partition it, write all -the data (probably by means of loop-back mounting the image or parts of it), -etc. The end result should be a sparse file where mapped areas represent useful -parts of the image and holes represent useless parts of the image, which do not -have to be copied when copying the image to the target device. - -This module uses the FIBMAP ioctl to detect holes. """ - -# Disable the following pylint recommendations: -# * Too many instance attributes - R0902 -# * Too few public methods - R0903 -# pylint: disable=R0902,R0903 - -import hashlib -from mic.utils.misc import human_size -from mic.utils import Fiemap - -# The bmap format version we generate -SUPPORTED_BMAP_VERSION = "1.3" - -_BMAP_START_TEMPLATE = \ -"""<?xml version="1.0" ?> -<!-- This file contains the block map for an image file, which is basically - a list of useful (mapped) block numbers in the image file. In other words, - it lists only those blocks which contain data (boot sector, partition - table, file-system metadata, files, directories, extents, etc). These - blocks have to be copied to the target device. The other blocks do not - contain any useful data and do not have to be copied to the target - device. - - The block map an optimization which allows to copy or flash the image to - the image quicker than copying of flashing the entire image. This is - because with bmap less data is copied: <MappedBlocksCount> blocks instead - of <BlocksCount> blocks. - - Besides the machine-readable data, this file contains useful commentaries - which contain human-readable information like image size, percentage of - mapped data, etc. - - The 'version' attribute is the block map file format version in the - 'major.minor' format. The version major number is increased whenever an - incompatible block map format change is made. The minor number changes - in case of minor backward-compatible changes. --> - -<bmap version="%s"> - <!-- Image size in bytes: %s --> - <ImageSize> %u </ImageSize> - - <!-- Size of a block in bytes --> - <BlockSize> %u </BlockSize> - - <!-- Count of blocks in the image file --> - <BlocksCount> %u </BlocksCount> - -""" - -class Error(Exception): - """ A class for exceptions generated by this module. We currently support - only one type of exceptions, and we basically throw human-readable problem - description in case of errors. """ - pass - -class BmapCreate: - """ This class implements the bmap creation functionality. To generate a - bmap for an image (which is supposedly a sparse file), you should first - create an instance of 'BmapCreate' and provide: - - * full path or a file-like object of the image to create bmap for - * full path or a file object to use for writing the results to - - Then you should invoke the 'generate()' method of this class. It will use - the FIEMAP ioctl to generate the bmap. """ - - def _open_image_file(self): - """ Open the image file. """ - - try: - self._f_image = open(self._image_path, 'rb') - except IOError as err: - raise Error("cannot open image file '%s': %s" \ - % (self._image_path, err)) - - self._f_image_needs_close = True - - def _open_bmap_file(self): - """ Open the bmap file. """ - - try: - self._f_bmap = open(self._bmap_path, 'w+') - except IOError as err: - raise Error("cannot open bmap file '%s': %s" \ - % (self._bmap_path, err)) - - self._f_bmap_needs_close = True - - def __init__(self, image, bmap): - """ Initialize a class instance: - * image - full path or a file-like object of the image to create bmap - for - * bmap - full path or a file object to use for writing the resulting - bmap to """ - - self.image_size = None - self.image_size_human = None - self.block_size = None - self.blocks_cnt = None - self.mapped_cnt = None - self.mapped_size = None - self.mapped_size_human = None - self.mapped_percent = None - - self._mapped_count_pos1 = None - self._mapped_count_pos2 = None - self._sha1_pos = None - - self._f_image_needs_close = False - self._f_bmap_needs_close = False - - if hasattr(image, "read"): - self._f_image = image - self._image_path = image.name - else: - self._image_path = image - self._open_image_file() - - if hasattr(bmap, "read"): - self._f_bmap = bmap - self._bmap_path = bmap.name - else: - self._bmap_path = bmap - self._open_bmap_file() - - self.fiemap = Fiemap.Fiemap(self._f_image) - - self.image_size = self.fiemap.image_size - self.image_size_human = human_size(self.image_size) - if self.image_size == 0: - raise Error("cannot generate bmap for zero-sized image file '%s'" \ - % self._image_path) - - self.block_size = self.fiemap.block_size - self.blocks_cnt = self.fiemap.blocks_cnt - - def _bmap_file_start(self): - """ A helper function which generates the starting contents of the - block map file: the header comment, image size, block size, etc. """ - - # We do not know the amount of mapped blocks at the moment, so just put - # whitespaces instead of real numbers. Assume the longest possible - # numbers. - mapped_count = ' ' * len(str(self.image_size)) - mapped_size_human = ' ' * len(self.image_size_human) - - xml = _BMAP_START_TEMPLATE \ - % (SUPPORTED_BMAP_VERSION, self.image_size_human, - self.image_size, self.block_size, self.blocks_cnt) - xml += " <!-- Count of mapped blocks: " - - self._f_bmap.write(xml) - self._mapped_count_pos1 = self._f_bmap.tell() - - # Just put white-spaces instead of real information about mapped blocks - xml = "%s or %.1f -->\n" % (mapped_size_human, 100.0) - xml += " <MappedBlocksCount> " - - self._f_bmap.write(xml) - self._mapped_count_pos2 = self._f_bmap.tell() - - xml = "%s </MappedBlocksCount>\n\n" % mapped_count - - # pylint: disable=C0301 - xml += " <!-- The checksum of this bmap file. When it is calculated, the value of\n" - xml += " the SHA1 checksum has be zeoro (40 ASCII \"0\" symbols). -->\n" - xml += " <BmapFileSHA1> " - - self._f_bmap.write(xml) - self._sha1_pos = self._f_bmap.tell() - - xml = "0" * 40 + " </BmapFileSHA1>\n\n" - xml += " <!-- The block map which consists of elements which may either be a\n" - xml += " range of blocks or a single block. The 'sha1' attribute (if present)\n" - xml += " is the SHA1 checksum of this blocks range. -->\n" - xml += " <BlockMap>\n" - # pylint: enable=C0301 - - self._f_bmap.write(xml) - - def _bmap_file_end(self): - """ A helper function which generates the final parts of the block map - file: the ending tags and the information about the amount of mapped - blocks. """ - - xml = " </BlockMap>\n" - xml += "</bmap>\n" - - self._f_bmap.write(xml) - - self._f_bmap.seek(self._mapped_count_pos1) - self._f_bmap.write("%s or %.1f%%" % \ - (self.mapped_size_human, self.mapped_percent)) - - self._f_bmap.seek(self._mapped_count_pos2) - self._f_bmap.write("%u" % self.mapped_cnt) - - self._f_bmap.seek(0) - sha1 = hashlib.sha1(self._f_bmap.read()).hexdigest() - self._f_bmap.seek(self._sha1_pos) - self._f_bmap.write("%s" % sha1) - - def _calculate_sha1(self, first, last): - """ A helper function which calculates SHA1 checksum for the range of - blocks of the image file: from block 'first' to block 'last'. """ - - start = first * self.block_size - end = (last + 1) * self.block_size - - self._f_image.seek(start) - hash_obj = hashlib.new("sha1") - - chunk_size = 1024*1024 - to_read = end - start - read = 0 - - while read < to_read: - if read + chunk_size > to_read: - chunk_size = to_read - read - chunk = self._f_image.read(chunk_size) - hash_obj.update(chunk) - read += chunk_size - - return hash_obj.hexdigest() - - def generate(self, include_checksums = True): - """ Generate bmap for the image file. If 'include_checksums' is 'True', - also generate SHA1 checksums for block ranges. """ - - # Save image file position in order to restore it at the end - image_pos = self._f_image.tell() - - self._bmap_file_start() - - # Generate the block map and write it to the XML block map - # file as we go. - self.mapped_cnt = 0 - for first, last in self.fiemap.get_mapped_ranges(0, self.blocks_cnt): - self.mapped_cnt += last - first + 1 - if include_checksums: - sha1 = self._calculate_sha1(first, last) - sha1 = " sha1=\"%s\"" % sha1 - else: - sha1 = "" - - if first != last: - self._f_bmap.write(" <Range%s> %s-%s </Range>\n" \ - % (sha1, first, last)) - else: - self._f_bmap.write(" <Range%s> %s </Range>\n" \ - % (sha1, first)) - - self.mapped_size = self.mapped_cnt * self.block_size - self.mapped_size_human = human_size(self.mapped_size) - self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt - - self._bmap_file_end() - - try: - self._f_bmap.flush() - except IOError as err: - raise Error("cannot flush the bmap file '%s': %s" \ - % (self._bmap_path, err)) - - self._f_image.seek(image_pos) - - def __del__(self): - """ The class destructor which closes the opened files. """ - - if self._f_image_needs_close: - self._f_image.close() - if self._f_bmap_needs_close: - self._f_bmap.close() diff --git a/scripts/lib/mic/utils/Fiemap.py b/scripts/lib/mic/utils/Fiemap.py deleted file mode 100644 index f2db6ff0b8..0000000000 --- a/scripts/lib/mic/utils/Fiemap.py +++ /dev/null @@ -1,252 +0,0 @@ -""" This module implements python API for the FIEMAP ioctl. The FIEMAP ioctl -allows to find holes and mapped areas in a file. """ - -# Note, a lot of code in this module is not very readable, because it deals -# with the rather complex FIEMAP ioctl. To understand the code, you need to -# know the FIEMAP interface, which is documented in the -# Documentation/filesystems/fiemap.txt file in the Linux kernel sources. - -# Disable the following pylint recommendations: -# * Too many instance attributes (R0902) -# pylint: disable=R0902 - -import os -import struct -import array -import fcntl -from mic.utils.misc import get_block_size - -# Format string for 'struct fiemap' -_FIEMAP_FORMAT = "=QQLLLL" -# sizeof(struct fiemap) -_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT) -# Format string for 'struct fiemap_extent' -_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL" -# sizeof(struct fiemap_extent) -_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT) -# The FIEMAP ioctl number -_FIEMAP_IOCTL = 0xC020660B - -# Minimum buffer which is required for 'class Fiemap' to operate -MIN_BUFFER_SIZE = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE -# The default buffer size for 'class Fiemap' -DEFAULT_BUFFER_SIZE = 256 * 1024 - -class Error(Exception): - """ A class for exceptions generated by this module. We currently support - only one type of exceptions, and we basically throw human-readable problem - description in case of errors. """ - pass - -class Fiemap: - """ This class provides API to the FIEMAP ioctl. Namely, it allows to - iterate over all mapped blocks and over all holes. """ - - def _open_image_file(self): - """ Open the image file. """ - - try: - self._f_image = open(self._image_path, 'rb') - except IOError as err: - raise Error("cannot open image file '%s': %s" \ - % (self._image_path, err)) - - self._f_image_needs_close = True - - def __init__(self, image, buf_size = DEFAULT_BUFFER_SIZE): - """ Initialize a class instance. The 'image' argument is full path to - the file to operate on, or a file object to operate on. - - The 'buf_size' argument is the size of the buffer for 'struct - fiemap_extent' elements which will be used when invoking the FIEMAP - ioctl. The larger is the buffer, the less times the FIEMAP ioctl will - be invoked. """ - - self._f_image_needs_close = False - - if hasattr(image, "fileno"): - self._f_image = image - self._image_path = image.name - else: - self._image_path = image - self._open_image_file() - - # Validate 'buf_size' - if buf_size < MIN_BUFFER_SIZE: - raise Error("too small buffer (%d bytes), minimum is %d bytes" \ - % (buf_size, MIN_BUFFER_SIZE)) - - # How many 'struct fiemap_extent' elements fit the buffer - buf_size -= _FIEMAP_SIZE - self._fiemap_extent_cnt = buf_size / _FIEMAP_EXTENT_SIZE - self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE - self._buf_size += _FIEMAP_SIZE - - # Allocate a mutable buffer for the FIEMAP ioctl - self._buf = array.array('B', [0] * self._buf_size) - - self.image_size = os.fstat(self._f_image.fileno()).st_size - - try: - self.block_size = get_block_size(self._f_image) - except IOError as err: - raise Error("cannot get block size for '%s': %s" \ - % (self._image_path, err)) - - self.blocks_cnt = self.image_size + self.block_size - 1 - self.blocks_cnt /= self.block_size - - # Synchronize the image file to make sure FIEMAP returns correct values - try: - self._f_image.flush() - except IOError as err: - raise Error("cannot flush image file '%s': %s" \ - % (self._image_path, err)) - try: - os.fsync(self._f_image.fileno()), - except OSError as err: - raise Error("cannot synchronize image file '%s': %s " \ - % (self._image_path, err.strerror)) - - # Check if the FIEMAP ioctl is supported - self.block_is_mapped(0) - - def __del__(self): - """ The class destructor which closes the opened files. """ - - if self._f_image_needs_close: - self._f_image.close() - - def _invoke_fiemap(self, block, count): - """ Invoke the FIEMAP ioctl for 'count' blocks of the file starting from - block number 'block'. - - The full result of the operation is stored in 'self._buf' on exit. - Returns the unpacked 'struct fiemap' data structure in form of a python - list (just like 'struct.upack()'). """ - - if block < 0 or block >= self.blocks_cnt: - raise Error("bad block number %d, should be within [0, %d]" \ - % (block, self.blocks_cnt)) - - # Initialize the 'struct fiemap' part of the buffer - struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size, - count * self.block_size, 0, 0, - self._fiemap_extent_cnt, 0) - - try: - fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1) - except IOError as err: - error_msg = "the FIEMAP ioctl failed for '%s': %s" \ - % (self._image_path, err) - if err.errno == os.errno.EPERM or err.errno == os.errno.EACCES: - # The FIEMAP ioctl was added in kernel version 2.6.28 in 2008 - error_msg += " (looks like your kernel does not support FIEMAP)" - - raise Error(error_msg) - - return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE]) - - def block_is_mapped(self, block): - """ This function returns 'True' if block number 'block' of the image - file is mapped and 'False' otherwise. """ - - struct_fiemap = self._invoke_fiemap(block, 1) - - # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field. - # If it contains zero, the block is not mapped, otherwise it is - # mapped. - return bool(struct_fiemap[3]) - - def block_is_unmapped(self, block): - """ This function returns 'True' if block number 'block' of the image - file is not mapped (hole) and 'False' otherwise. """ - - return not self.block_is_mapped(block) - - def _unpack_fiemap_extent(self, index): - """ Unpack a 'struct fiemap_extent' structure object number 'index' - from the internal 'self._buf' buffer. """ - - offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index - return struct.unpack(_FIEMAP_EXTENT_FORMAT, - self._buf[offset : offset + _FIEMAP_EXTENT_SIZE]) - - def _do_get_mapped_ranges(self, start, count): - """ Implements most the functionality for the 'get_mapped_ranges()' - generator: invokes the FIEMAP ioctl, walks through the mapped - extents and yields mapped block ranges. However, the ranges may be - consecutive (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' - simply merges them. """ - - block = start - while block < start + count: - struct_fiemap = self._invoke_fiemap(block, count) - - mapped_extents = struct_fiemap[3] - if mapped_extents == 0: - # No more mapped blocks - return - - extent = 0 - while extent < mapped_extents: - fiemap_extent = self._unpack_fiemap_extent(extent) - - # Start of the extent - extent_start = fiemap_extent[0] - # Starting block number of the extent - extent_block = extent_start / self.block_size - # Length of the extent - extent_len = fiemap_extent[2] - # Count of blocks in the extent - extent_count = extent_len / self.block_size - - # Extent length and offset have to be block-aligned - assert extent_start % self.block_size == 0 - assert extent_len % self.block_size == 0 - - if extent_block > start + count - 1: - return - - first = max(extent_block, block) - last = min(extent_block + extent_count, start + count) - 1 - yield (first, last) - - extent += 1 - - block = extent_block + extent_count - - def get_mapped_ranges(self, start, count): - """ A generator which yields ranges of mapped blocks in the file. The - ranges are tuples of 2 elements: [first, last], where 'first' is the - first mapped block and 'last' is the last mapped block. - - The ranges are yielded for the area of the file of size 'count' blocks, - starting from block 'start'. """ - - iterator = self._do_get_mapped_ranges(start, count) - - first_prev, last_prev = iterator.next() - - for first, last in iterator: - if last_prev == first - 1: - last_prev = last - else: - yield (first_prev, last_prev) - first_prev, last_prev = first, last - - yield (first_prev, last_prev) - - def get_unmapped_ranges(self, start, count): - """ Just like 'get_mapped_ranges()', but yields unmapped block ranges - instead (holes). """ - - hole_first = start - for first, last in self._do_get_mapped_ranges(start, count): - if first > hole_first: - yield (hole_first, first - 1) - - hole_first = last + 1 - - if hole_first < start + count: - yield (hole_first, start + count - 1) diff --git a/scripts/lib/mic/utils/__init__.py b/scripts/lib/mic/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 --- a/scripts/lib/mic/utils/__init__.py +++ /dev/null diff --git a/scripts/lib/mic/utils/cmdln.py b/scripts/lib/mic/utils/cmdln.py deleted file mode 100644 index b099473ee4..0000000000 --- a/scripts/lib/mic/utils/cmdln.py +++ /dev/null @@ -1,1586 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2002-2007 ActiveState Software Inc. -# License: MIT (see LICENSE.txt for license details) -# Author: Trent Mick -# Home: http://trentm.com/projects/cmdln/ - -"""An improvement on Python's standard cmd.py module. - -As with cmd.py, this module provides "a simple framework for writing -line-oriented command intepreters." This module provides a 'RawCmdln' -class that fixes some design flaws in cmd.Cmd, making it more scalable -and nicer to use for good 'cvs'- or 'svn'-style command line interfaces -or simple shells. And it provides a 'Cmdln' class that add -optparse-based option processing. Basically you use it like this: - - import cmdln - - class MySVN(cmdln.Cmdln): - name = "svn" - - @cmdln.alias('stat', 'st') - @cmdln.option('-v', '--verbose', action='store_true' - help='print verbose information') - def do_status(self, subcmd, opts, *paths): - print "handle 'svn status' command" - - #... - - if __name__ == "__main__": - shell = MySVN() - retval = shell.main() - sys.exit(retval) - -See the README.txt or <http://trentm.com/projects/cmdln/> for more -details. -""" - -__version_info__ = (1, 1, 2) -__version__ = '.'.join(map(str, __version_info__)) - -import os -import sys -import re -import cmd -import optparse -from pprint import pprint -import sys - - - - -#---- globals - -LOOP_ALWAYS, LOOP_NEVER, LOOP_IF_EMPTY = range(3) - -# An unspecified optional argument when None is a meaningful value. -_NOT_SPECIFIED = ("Not", "Specified") - -# Pattern to match a TypeError message from a call that -# failed because of incorrect number of arguments (see -# Python/getargs.c). -_INCORRECT_NUM_ARGS_RE = re.compile( - r"(takes [\w ]+ )(\d+)( arguments? \()(\d+)( given\))") - - - -#---- exceptions - -class CmdlnError(Exception): - """A cmdln.py usage error.""" - def __init__(self, msg): - self.msg = msg - def __str__(self): - return self.msg - -class CmdlnUserError(Exception): - """An error by a user of a cmdln-based tool/shell.""" - pass - - - -#---- public methods and classes - -def alias(*aliases): - """Decorator to add aliases for Cmdln.do_* command handlers. - - Example: - class MyShell(cmdln.Cmdln): - @cmdln.alias("!", "sh") - def do_shell(self, argv): - #...implement 'shell' command - """ - def decorate(f): - if not hasattr(f, "aliases"): - f.aliases = [] - f.aliases += aliases - return f - return decorate - - -class RawCmdln(cmd.Cmd): - """An improved (on cmd.Cmd) framework for building multi-subcommand - scripts (think "svn" & "cvs") and simple shells (think "pdb" and - "gdb"). - - A simple example: - - import cmdln - - class MySVN(cmdln.RawCmdln): - name = "svn" - - @cmdln.aliases('stat', 'st') - def do_status(self, argv): - print "handle 'svn status' command" - - if __name__ == "__main__": - shell = MySVN() - retval = shell.main() - sys.exit(retval) - - See <http://trentm.com/projects/cmdln> for more information. - """ - name = None # if unset, defaults basename(sys.argv[0]) - prompt = None # if unset, defaults to self.name+"> " - version = None # if set, default top-level options include --version - - # Default messages for some 'help' command error cases. - # They are interpolated with one arg: the command. - nohelp = "no help on '%s'" - unknowncmd = "unknown command: '%s'" - - helpindent = '' # string with which to indent help output - - def __init__(self, completekey='tab', - stdin=None, stdout=None, stderr=None): - """Cmdln(completekey='tab', stdin=None, stdout=None, stderr=None) - - The optional argument 'completekey' is the readline name of a - completion key; it defaults to the Tab key. If completekey is - not None and the readline module is available, command completion - is done automatically. - - The optional arguments 'stdin', 'stdout' and 'stderr' specify - alternate input, output and error output file objects; if not - specified, sys.* are used. - - If 'stdout' but not 'stderr' is specified, stdout is used for - error output. This is to provide least surprise for users used - to only the 'stdin' and 'stdout' options with cmd.Cmd. - """ - import sys - if self.name is None: - self.name = os.path.basename(sys.argv[0]) - if self.prompt is None: - self.prompt = self.name+"> " - self._name_str = self._str(self.name) - self._prompt_str = self._str(self.prompt) - if stdin is not None: - self.stdin = stdin - else: - self.stdin = sys.stdin - if stdout is not None: - self.stdout = stdout - else: - self.stdout = sys.stdout - if stderr is not None: - self.stderr = stderr - elif stdout is not None: - self.stderr = stdout - else: - self.stderr = sys.stderr - self.cmdqueue = [] - self.completekey = completekey - self.cmdlooping = False - - def get_optparser(self): - """Hook for subclasses to set the option parser for the - top-level command/shell. - - This option parser is used retrieved and used by `.main()' to - handle top-level options. - - The default implements a single '-h|--help' option. Sub-classes - can return None to have no options at the top-level. Typically - an instance of CmdlnOptionParser should be returned. - """ - version = (self.version is not None - and "%s %s" % (self._name_str, self.version) - or None) - return CmdlnOptionParser(self, version=version) - - def postoptparse(self): - """Hook method executed just after `.main()' parses top-level - options. - - When called `self.options' holds the results of the option parse. - """ - pass - - def main(self, argv=None, loop=LOOP_NEVER): - """A possible mainline handler for a script, like so: - - import cmdln - class MyCmd(cmdln.Cmdln): - name = "mycmd" - ... - - if __name__ == "__main__": - MyCmd().main() - - By default this will use sys.argv to issue a single command to - 'MyCmd', then exit. The 'loop' argument can be use to control - interactive shell behaviour. - - Arguments: - "argv" (optional, default sys.argv) is the command to run. - It must be a sequence, where the first element is the - command name and subsequent elements the args for that - command. - "loop" (optional, default LOOP_NEVER) is a constant - indicating if a command loop should be started (i.e. an - interactive shell). Valid values (constants on this module): - LOOP_ALWAYS start loop and run "argv", if any - LOOP_NEVER run "argv" (or .emptyline()) and exit - LOOP_IF_EMPTY run "argv", if given, and exit; - otherwise, start loop - """ - if argv is None: - import sys - argv = sys.argv - else: - argv = argv[:] # don't modify caller's list - - self.optparser = self.get_optparser() - if self.optparser: # i.e. optparser=None means don't process for opts - try: - self.options, args = self.optparser.parse_args(argv[1:]) - except CmdlnUserError, ex: - msg = "%s: %s\nTry '%s help' for info.\n"\ - % (self.name, ex, self.name) - self.stderr.write(self._str(msg)) - self.stderr.flush() - return 1 - except StopOptionProcessing, ex: - return 0 - else: - self.options, args = None, argv[1:] - self.postoptparse() - - if loop == LOOP_ALWAYS: - if args: - self.cmdqueue.append(args) - return self.cmdloop() - elif loop == LOOP_NEVER: - if args: - return self.cmd(args) - else: - return self.emptyline() - elif loop == LOOP_IF_EMPTY: - if args: - return self.cmd(args) - else: - return self.cmdloop() - - def cmd(self, argv): - """Run one command and exit. - - "argv" is the arglist for the command to run. argv[0] is the - command to run. If argv is an empty list then the - 'emptyline' handler is run. - - Returns the return value from the command handler. - """ - assert isinstance(argv, (list, tuple)), \ - "'argv' is not a sequence: %r" % argv - retval = None - try: - argv = self.precmd(argv) - retval = self.onecmd(argv) - self.postcmd(argv) - except: - if not self.cmdexc(argv): - raise - retval = 1 - return retval - - def _str(self, s): - """Safely convert the given str/unicode to a string for printing.""" - try: - return str(s) - except UnicodeError: - #XXX What is the proper encoding to use here? 'utf-8' seems - # to work better than "getdefaultencoding" (usually - # 'ascii'), on OS X at least. - #import sys - #return s.encode(sys.getdefaultencoding(), "replace") - return s.encode("utf-8", "replace") - - def cmdloop(self, intro=None): - """Repeatedly issue a prompt, accept input, parse into an argv, and - dispatch (via .precmd(), .onecmd() and .postcmd()), passing them - the argv. In other words, start a shell. - - "intro" (optional) is a introductory message to print when - starting the command loop. This overrides the class - "intro" attribute, if any. - """ - self.cmdlooping = True - self.preloop() - if self.use_rawinput and self.completekey: - try: - import readline - self.old_completer = readline.get_completer() - readline.set_completer(self.complete) - readline.parse_and_bind(self.completekey+": complete") - except ImportError: - pass - try: - if intro is None: - intro = self.intro - if intro: - intro_str = self._str(intro) - self.stdout.write(intro_str+'\n') - self.stop = False - retval = None - while not self.stop: - if self.cmdqueue: - argv = self.cmdqueue.pop(0) - assert isinstance(argv, (list, tuple)), \ - "item on 'cmdqueue' is not a sequence: %r" % argv - else: - if self.use_rawinput: - try: - line = raw_input(self._prompt_str) - except EOFError: - line = 'EOF' - else: - self.stdout.write(self._prompt_str) - self.stdout.flush() - line = self.stdin.readline() - if not len(line): - line = 'EOF' - else: - line = line[:-1] # chop '\n' - argv = line2argv(line) - try: - argv = self.precmd(argv) - retval = self.onecmd(argv) - self.postcmd(argv) - except: - if not self.cmdexc(argv): - raise - retval = 1 - self.lastretval = retval - self.postloop() - finally: - if self.use_rawinput and self.completekey: - try: - import readline - readline.set_completer(self.old_completer) - except ImportError: - pass - self.cmdlooping = False - return retval - - def precmd(self, argv): - """Hook method executed just before the command argv is - interpreted, but after the input prompt is generated and issued. - - "argv" is the cmd to run. - - Returns an argv to run (i.e. this method can modify the command - to run). - """ - return argv - - def postcmd(self, argv): - """Hook method executed just after a command dispatch is finished. - - "argv" is the command that was run. - """ - pass - - def cmdexc(self, argv): - """Called if an exception is raised in any of precmd(), onecmd(), - or postcmd(). If True is returned, the exception is deemed to have - been dealt with. Otherwise, the exception is re-raised. - - The default implementation handles CmdlnUserError's, which - typically correspond to user error in calling commands (as - opposed to programmer error in the design of the script using - cmdln.py). - """ - import sys - type, exc, traceback = sys.exc_info() - if isinstance(exc, CmdlnUserError): - msg = "%s %s: %s\nTry '%s help %s' for info.\n"\ - % (self.name, argv[0], exc, self.name, argv[0]) - self.stderr.write(self._str(msg)) - self.stderr.flush() - return True - - def onecmd(self, argv): - if not argv: - return self.emptyline() - self.lastcmd = argv - cmdname = self._get_canonical_cmd_name(argv[0]) - if cmdname: - handler = self._get_cmd_handler(cmdname) - if handler: - return self._dispatch_cmd(handler, argv) - return self.default(argv) - - def _dispatch_cmd(self, handler, argv): - return handler(argv) - - def default(self, argv): - """Hook called to handle a command for which there is no handler. - - "argv" is the command and arguments to run. - - The default implementation writes and error message to stderr - and returns an error exit status. - - Returns a numeric command exit status. - """ - errmsg = self._str(self.unknowncmd % (argv[0],)) - if self.cmdlooping: - self.stderr.write(errmsg+"\n") - else: - self.stderr.write("%s: %s\nTry '%s help' for info.\n" - % (self._name_str, errmsg, self._name_str)) - self.stderr.flush() - return 1 - - def parseline(self, line): - # This is used by Cmd.complete (readline completer function) to - # massage the current line buffer before completion processing. - # We override to drop special '!' handling. - line = line.strip() - if not line: - return None, None, line - elif line[0] == '?': - line = 'help ' + line[1:] - i, n = 0, len(line) - while i < n and line[i] in self.identchars: i = i+1 - cmd, arg = line[:i], line[i:].strip() - return cmd, arg, line - - def helpdefault(self, cmd, known): - """Hook called to handle help on a command for which there is no - help handler. - - "cmd" is the command name on which help was requested. - "known" is a boolean indicating if this command is known - (i.e. if there is a handler for it). - - Returns a return code. - """ - if known: - msg = self._str(self.nohelp % (cmd,)) - if self.cmdlooping: - self.stderr.write(msg + '\n') - else: - self.stderr.write("%s: %s\n" % (self.name, msg)) - else: - msg = self.unknowncmd % (cmd,) - if self.cmdlooping: - self.stderr.write(msg + '\n') - else: - self.stderr.write("%s: %s\n" - "Try '%s help' for info.\n" - % (self.name, msg, self.name)) - self.stderr.flush() - return 1 - - def do_help(self, argv): - """${cmd_name}: give detailed help on a specific sub-command - - Usage: - ${name} help [COMMAND] - """ - if len(argv) > 1: # asking for help on a particular command - doc = None - cmdname = self._get_canonical_cmd_name(argv[1]) or argv[1] - if not cmdname: - return self.helpdefault(argv[1], False) - else: - helpfunc = getattr(self, "help_"+cmdname, None) - if helpfunc: - doc = helpfunc() - else: - handler = self._get_cmd_handler(cmdname) - if handler: - doc = handler.__doc__ - if doc is None: - return self.helpdefault(argv[1], handler != None) - else: # bare "help" command - doc = self.__class__.__doc__ # try class docstring - if doc is None: - # Try to provide some reasonable useful default help. - if self.cmdlooping: prefix = "" - else: prefix = self.name+' ' - doc = """Usage: - %sCOMMAND [ARGS...] - %shelp [COMMAND] - - ${option_list} - ${command_list} - ${help_list} - """ % (prefix, prefix) - cmdname = None - - if doc: # *do* have help content, massage and print that - doc = self._help_reindent(doc) - doc = self._help_preprocess(doc, cmdname) - doc = doc.rstrip() + '\n' # trim down trailing space - self.stdout.write(self._str(doc)) - self.stdout.flush() - do_help.aliases = ["?"] - - def _help_reindent(self, help, indent=None): - """Hook to re-indent help strings before writing to stdout. - - "help" is the help content to re-indent - "indent" is a string with which to indent each line of the - help content after normalizing. If unspecified or None - then the default is use: the 'self.helpindent' class - attribute. By default this is the empty string, i.e. - no indentation. - - By default, all common leading whitespace is removed and then - the lot is indented by 'self.helpindent'. When calculating the - common leading whitespace the first line is ignored -- hence - help content for Conan can be written as follows and have the - expected indentation: - - def do_crush(self, ...): - '''${cmd_name}: crush your enemies, see them driven before you... - - c.f. Conan the Barbarian''' - """ - if indent is None: - indent = self.helpindent - lines = help.splitlines(0) - _dedentlines(lines, skip_first_line=True) - lines = [(indent+line).rstrip() for line in lines] - return '\n'.join(lines) - - def _help_preprocess(self, help, cmdname): - """Hook to preprocess a help string before writing to stdout. - - "help" is the help string to process. - "cmdname" is the canonical sub-command name for which help - is being given, or None if the help is not specific to a - command. - - By default the following template variables are interpolated in - help content. (Note: these are similar to Python 2.4's - string.Template interpolation but not quite.) - - ${name} - The tool's/shell's name, i.e. 'self.name'. - ${option_list} - A formatted table of options for this shell/tool. - ${command_list} - A formatted table of available sub-commands. - ${help_list} - A formatted table of additional help topics (i.e. 'help_*' - methods with no matching 'do_*' method). - ${cmd_name} - The name (and aliases) for this sub-command formatted as: - "NAME (ALIAS1, ALIAS2, ...)". - ${cmd_usage} - A formatted usage block inferred from the command function - signature. - ${cmd_option_list} - A formatted table of options for this sub-command. (This is - only available for commands using the optparse integration, - i.e. using @cmdln.option decorators or manually setting the - 'optparser' attribute on the 'do_*' method.) - - Returns the processed help. - """ - preprocessors = { - "${name}": self._help_preprocess_name, - "${option_list}": self._help_preprocess_option_list, - "${command_list}": self._help_preprocess_command_list, - "${help_list}": self._help_preprocess_help_list, - "${cmd_name}": self._help_preprocess_cmd_name, - "${cmd_usage}": self._help_preprocess_cmd_usage, - "${cmd_option_list}": self._help_preprocess_cmd_option_list, - } - - for marker, preprocessor in preprocessors.items(): - if marker in help: - help = preprocessor(help, cmdname) - return help - - def _help_preprocess_name(self, help, cmdname=None): - return help.replace("${name}", self.name) - - def _help_preprocess_option_list(self, help, cmdname=None): - marker = "${option_list}" - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - if self.optparser: - # Setup formatting options and format. - # - Indentation of 4 is better than optparse default of 2. - # C.f. Damian Conway's discussion of this in Perl Best - # Practices. - self.optparser.formatter.indent_increment = 4 - self.optparser.formatter.current_indent = indent_width - block = self.optparser.format_option_help() + '\n' - else: - block = "" - - help = help.replace(indent+marker+suffix, block, 1) - return help - - - def _help_preprocess_command_list(self, help, cmdname=None): - marker = "${command_list}" - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - # Find any aliases for commands. - token2canonical = self._get_canonical_map() - aliases = {} - for token, cmdname in token2canonical.items(): - if token == cmdname: continue - aliases.setdefault(cmdname, []).append(token) - - # Get the list of (non-hidden) commands and their - # documentation, if any. - cmdnames = {} # use a dict to strip duplicates - for attr in self.get_names(): - if attr.startswith("do_"): - cmdnames[attr[3:]] = True - cmdnames = cmdnames.keys() - cmdnames.sort() - linedata = [] - for cmdname in cmdnames: - if aliases.get(cmdname): - a = aliases[cmdname] - a.sort() - cmdstr = "%s (%s)" % (cmdname, ", ".join(a)) - else: - cmdstr = cmdname - doc = None - try: - helpfunc = getattr(self, 'help_'+cmdname) - except AttributeError: - handler = self._get_cmd_handler(cmdname) - if handler: - doc = handler.__doc__ - else: - doc = helpfunc() - - # Strip "${cmd_name}: " from the start of a command's doc. Best - # practice dictates that command help strings begin with this, but - # it isn't at all wanted for the command list. - to_strip = "${cmd_name}:" - if doc and doc.startswith(to_strip): - #log.debug("stripping %r from start of %s's help string", - # to_strip, cmdname) - doc = doc[len(to_strip):].lstrip() - linedata.append( (cmdstr, doc) ) - - if linedata: - subindent = indent + ' '*4 - lines = _format_linedata(linedata, subindent, indent_width+4) - block = indent + "Commands:\n" \ - + '\n'.join(lines) + "\n\n" - help = help.replace(indent+marker+suffix, block, 1) - return help - - def _gen_names_and_attrs(self): - # Inheritance says we have to look in class and - # base classes; order is not important. - names = [] - classes = [self.__class__] - while classes: - aclass = classes.pop(0) - if aclass.__bases__: - classes = classes + list(aclass.__bases__) - for name in dir(aclass): - yield (name, getattr(aclass, name)) - - def _help_preprocess_help_list(self, help, cmdname=None): - marker = "${help_list}" - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - # Determine the additional help topics, if any. - helpnames = {} - token2cmdname = self._get_canonical_map() - for attrname, attr in self._gen_names_and_attrs(): - if not attrname.startswith("help_"): continue - helpname = attrname[5:] - if helpname not in token2cmdname: - helpnames[helpname] = attr - - if helpnames: - linedata = [(n, a.__doc__ or "") for n, a in helpnames.items()] - linedata.sort() - - subindent = indent + ' '*4 - lines = _format_linedata(linedata, subindent, indent_width+4) - block = (indent - + "Additional help topics (run `%s help TOPIC'):\n" % self.name - + '\n'.join(lines) - + "\n\n") - else: - block = '' - help = help.replace(indent+marker+suffix, block, 1) - return help - - def _help_preprocess_cmd_name(self, help, cmdname=None): - marker = "${cmd_name}" - handler = self._get_cmd_handler(cmdname) - if not handler: - raise CmdlnError("cannot preprocess '%s' into help string: " - "could not find command handler for %r" - % (marker, cmdname)) - s = cmdname - if hasattr(handler, "aliases"): - s += " (%s)" % (", ".join(handler.aliases)) - help = help.replace(marker, s) - return help - - #TODO: this only makes sense as part of the Cmdln class. - # Add hooks to add help preprocessing template vars and put - # this one on that class. - def _help_preprocess_cmd_usage(self, help, cmdname=None): - marker = "${cmd_usage}" - handler = self._get_cmd_handler(cmdname) - if not handler: - raise CmdlnError("cannot preprocess '%s' into help string: " - "could not find command handler for %r" - % (marker, cmdname)) - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - # Extract the introspection bits we need. - func = handler.im_func - if func.func_defaults: - func_defaults = list(func.func_defaults) - else: - func_defaults = [] - co_argcount = func.func_code.co_argcount - co_varnames = func.func_code.co_varnames - co_flags = func.func_code.co_flags - CO_FLAGS_ARGS = 4 - CO_FLAGS_KWARGS = 8 - - # Adjust argcount for possible *args and **kwargs arguments. - argcount = co_argcount - if co_flags & CO_FLAGS_ARGS: argcount += 1 - if co_flags & CO_FLAGS_KWARGS: argcount += 1 - - # Determine the usage string. - usage = "%s %s" % (self.name, cmdname) - if argcount <= 2: # handler ::= do_FOO(self, argv) - usage += " [ARGS...]" - elif argcount >= 3: # handler ::= do_FOO(self, subcmd, opts, ...) - argnames = list(co_varnames[3:argcount]) - tail = "" - if co_flags & CO_FLAGS_KWARGS: - name = argnames.pop(-1) - import warnings - # There is no generally accepted mechanism for passing - # keyword arguments from the command line. Could - # *perhaps* consider: arg=value arg2=value2 ... - warnings.warn("argument '**%s' on '%s.%s' command " - "handler will never get values" - % (name, self.__class__.__name__, - func.func_name)) - if co_flags & CO_FLAGS_ARGS: - name = argnames.pop(-1) - tail = "[%s...]" % name.upper() - while func_defaults: - func_defaults.pop(-1) - name = argnames.pop(-1) - tail = "[%s%s%s]" % (name.upper(), (tail and ' ' or ''), tail) - while argnames: - name = argnames.pop(-1) - tail = "%s %s" % (name.upper(), tail) - usage += ' ' + tail - - block_lines = [ - self.helpindent + "Usage:", - self.helpindent + ' '*4 + usage - ] - block = '\n'.join(block_lines) + '\n\n' - - help = help.replace(indent+marker+suffix, block, 1) - return help - - #TODO: this only makes sense as part of the Cmdln class. - # Add hooks to add help preprocessing template vars and put - # this one on that class. - def _help_preprocess_cmd_option_list(self, help, cmdname=None): - marker = "${cmd_option_list}" - handler = self._get_cmd_handler(cmdname) - if not handler: - raise CmdlnError("cannot preprocess '%s' into help string: " - "could not find command handler for %r" - % (marker, cmdname)) - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - if hasattr(handler, "optparser"): - # Setup formatting options and format. - # - Indentation of 4 is better than optparse default of 2. - # C.f. Damian Conway's discussion of this in Perl Best - # Practices. - handler.optparser.formatter.indent_increment = 4 - handler.optparser.formatter.current_indent = indent_width - block = handler.optparser.format_option_help() + '\n' - else: - block = "" - - help = help.replace(indent+marker+suffix, block, 1) - return help - - def _get_canonical_cmd_name(self, token): - map = self._get_canonical_map() - return map.get(token, None) - - def _get_canonical_map(self): - """Return a mapping of available command names and aliases to - their canonical command name. - """ - cacheattr = "_token2canonical" - if not hasattr(self, cacheattr): - # Get the list of commands and their aliases, if any. - token2canonical = {} - cmd2funcname = {} # use a dict to strip duplicates - for attr in self.get_names(): - if attr.startswith("do_"): cmdname = attr[3:] - elif attr.startswith("_do_"): cmdname = attr[4:] - else: - continue - cmd2funcname[cmdname] = attr - token2canonical[cmdname] = cmdname - for cmdname, funcname in cmd2funcname.items(): # add aliases - func = getattr(self, funcname) - aliases = getattr(func, "aliases", []) - for alias in aliases: - if alias in cmd2funcname: - import warnings - warnings.warn("'%s' alias for '%s' command conflicts " - "with '%s' handler" - % (alias, cmdname, cmd2funcname[alias])) - continue - token2canonical[alias] = cmdname - setattr(self, cacheattr, token2canonical) - return getattr(self, cacheattr) - - def _get_cmd_handler(self, cmdname): - handler = None - try: - handler = getattr(self, 'do_' + cmdname) - except AttributeError: - try: - # Private command handlers begin with "_do_". - handler = getattr(self, '_do_' + cmdname) - except AttributeError: - pass - return handler - - def _do_EOF(self, argv): - # Default EOF handler - # Note: an actual EOF is redirected to this command. - #TODO: separate name for this. Currently it is available from - # command-line. Is that okay? - self.stdout.write('\n') - self.stdout.flush() - self.stop = True - - def emptyline(self): - # Different from cmd.Cmd: don't repeat the last command for an - # emptyline. - if self.cmdlooping: - pass - else: - return self.do_help(["help"]) - - -#---- optparse.py extension to fix (IMO) some deficiencies -# -# See the class _OptionParserEx docstring for details. -# - -class StopOptionProcessing(Exception): - """Indicate that option *and argument* processing should stop - cleanly. This is not an error condition. It is similar in spirit to - StopIteration. This is raised by _OptionParserEx's default "help" - and "version" option actions and can be raised by custom option - callbacks too. - - Hence the typical CmdlnOptionParser (a subclass of _OptionParserEx) - usage is: - - parser = CmdlnOptionParser(mycmd) - parser.add_option("-f", "--force", dest="force") - ... - try: - opts, args = parser.parse_args() - except StopOptionProcessing: - # normal termination, "--help" was probably given - sys.exit(0) - """ - -class _OptionParserEx(optparse.OptionParser): - """An optparse.OptionParser that uses exceptions instead of sys.exit. - - This class is an extension of optparse.OptionParser that differs - as follows: - - Correct (IMO) the default OptionParser error handling to never - sys.exit(). Instead OptParseError exceptions are passed through. - - Add the StopOptionProcessing exception (a la StopIteration) to - indicate normal termination of option processing. - See StopOptionProcessing's docstring for details. - - I'd also like to see the following in the core optparse.py, perhaps - as a RawOptionParser which would serve as a base class for the more - generally used OptionParser (that works as current): - - Remove the implicit addition of the -h|--help and --version - options. They can get in the way (e.g. if want '-?' and '-V' for - these as well) and it is not hard to do: - optparser.add_option("-h", "--help", action="help") - optparser.add_option("--version", action="version") - These are good practices, just not valid defaults if they can - get in the way. - """ - def error(self, msg): - raise optparse.OptParseError(msg) - - def exit(self, status=0, msg=None): - if status == 0: - raise StopOptionProcessing(msg) - else: - #TODO: don't lose status info here - raise optparse.OptParseError(msg) - - - -#---- optparse.py-based option processing support - -class CmdlnOptionParser(_OptionParserEx): - """An optparse.OptionParser class more appropriate for top-level - Cmdln options. For parsing of sub-command options, see - SubCmdOptionParser. - - Changes: - - disable_interspersed_args() by default, because a Cmdln instance - has sub-commands which may themselves have options. - - Redirect print_help() to the Cmdln.do_help() which is better - equiped to handle the "help" action. - - error() will raise a CmdlnUserError: OptionParse.error() is meant - to be called for user errors. Raising a well-known error here can - make error handling clearer. - - Also see the changes in _OptionParserEx. - """ - def __init__(self, cmdln, **kwargs): - self.cmdln = cmdln - kwargs["prog"] = self.cmdln.name - _OptionParserEx.__init__(self, **kwargs) - self.disable_interspersed_args() - - def print_help(self, file=None): - self.cmdln.onecmd(["help"]) - - def error(self, msg): - raise CmdlnUserError(msg) - - -class SubCmdOptionParser(_OptionParserEx): - def set_cmdln_info(self, cmdln, subcmd): - """Called by Cmdln to pass relevant info about itself needed - for print_help(). - """ - self.cmdln = cmdln - self.subcmd = subcmd - - def print_help(self, file=None): - self.cmdln.onecmd(["help", self.subcmd]) - - def error(self, msg): - raise CmdlnUserError(msg) - - -def option(*args, **kwargs): - """Decorator to add an option to the optparser argument of a Cmdln - subcommand. - - Example: - class MyShell(cmdln.Cmdln): - @cmdln.option("-f", "--force", help="force removal") - def do_remove(self, subcmd, opts, *args): - #... - """ - #XXX Is there a possible optimization for many options to not have a - # large stack depth here? - def decorate(f): - if not hasattr(f, "optparser"): - f.optparser = SubCmdOptionParser() - f.optparser.add_option(*args, **kwargs) - return f - return decorate - - -class Cmdln(RawCmdln): - """An improved (on cmd.Cmd) framework for building multi-subcommand - scripts (think "svn" & "cvs") and simple shells (think "pdb" and - "gdb"). - - A simple example: - - import cmdln - - class MySVN(cmdln.Cmdln): - name = "svn" - - @cmdln.aliases('stat', 'st') - @cmdln.option('-v', '--verbose', action='store_true' - help='print verbose information') - def do_status(self, subcmd, opts, *paths): - print "handle 'svn status' command" - - #... - - if __name__ == "__main__": - shell = MySVN() - retval = shell.main() - sys.exit(retval) - - 'Cmdln' extends 'RawCmdln' by providing optparse option processing - integration. See this class' _dispatch_cmd() docstring and - <http://trentm.com/projects/cmdln> for more information. - """ - def _dispatch_cmd(self, handler, argv): - """Introspect sub-command handler signature to determine how to - dispatch the command. The raw handler provided by the base - 'RawCmdln' class is still supported: - - def do_foo(self, argv): - # 'argv' is the vector of command line args, argv[0] is - # the command name itself (i.e. "foo" or an alias) - pass - - In addition, if the handler has more than 2 arguments option - processing is automatically done (using optparse): - - @cmdln.option('-v', '--verbose', action='store_true') - def do_bar(self, subcmd, opts, *args): - # subcmd = <"bar" or an alias> - # opts = <an optparse.Values instance> - if opts.verbose: - print "lots of debugging output..." - # args = <tuple of arguments> - for arg in args: - bar(arg) - - TODO: explain that "*args" can be other signatures as well. - - The `cmdln.option` decorator corresponds to an `add_option()` - method call on an `optparse.OptionParser` instance. - - You can declare a specific number of arguments: - - @cmdln.option('-v', '--verbose', action='store_true') - def do_bar2(self, subcmd, opts, bar_one, bar_two): - #... - - and an appropriate error message will be raised/printed if the - command is called with a different number of args. - """ - co_argcount = handler.im_func.func_code.co_argcount - if co_argcount == 2: # handler ::= do_foo(self, argv) - return handler(argv) - elif co_argcount >= 3: # handler ::= do_foo(self, subcmd, opts, ...) - try: - optparser = handler.optparser - except AttributeError: - optparser = handler.im_func.optparser = SubCmdOptionParser() - assert isinstance(optparser, SubCmdOptionParser) - optparser.set_cmdln_info(self, argv[0]) - try: - opts, args = optparser.parse_args(argv[1:]) - except StopOptionProcessing: - #TODO: this doesn't really fly for a replacement of - # optparse.py behaviour, does it? - return 0 # Normal command termination - - try: - return handler(argv[0], opts, *args) - except TypeError, ex: - # Some TypeError's are user errors: - # do_foo() takes at least 4 arguments (3 given) - # do_foo() takes at most 5 arguments (6 given) - # do_foo() takes exactly 5 arguments (6 given) - # Raise CmdlnUserError for these with a suitably - # massaged error message. - import sys - tb = sys.exc_info()[2] # the traceback object - if tb.tb_next is not None: - # If the traceback is more than one level deep, then the - # TypeError do *not* happen on the "handler(...)" call - # above. In that we don't want to handle it specially - # here: it would falsely mask deeper code errors. - raise - msg = ex.args[0] - match = _INCORRECT_NUM_ARGS_RE.search(msg) - if match: - msg = list(match.groups()) - msg[1] = int(msg[1]) - 3 - if msg[1] == 1: - msg[2] = msg[2].replace("arguments", "argument") - msg[3] = int(msg[3]) - 3 - msg = ''.join(map(str, msg)) - raise CmdlnUserError(msg) - else: - raise - else: - raise CmdlnError("incorrect argcount for %s(): takes %d, must " - "take 2 for 'argv' signature or 3+ for 'opts' " - "signature" % (handler.__name__, co_argcount)) - - - -#---- internal support functions - -def _format_linedata(linedata, indent, indent_width): - """Format specific linedata into a pleasant layout. - - "linedata" is a list of 2-tuples of the form: - (<item-display-string>, <item-docstring>) - "indent" is a string to use for one level of indentation - "indent_width" is a number of columns by which the - formatted data will be indented when printed. - - The <item-display-string> column is held to 15 columns. - """ - lines = [] - WIDTH = 78 - indent_width - SPACING = 2 - NAME_WIDTH_LOWER_BOUND = 13 - NAME_WIDTH_UPPER_BOUND = 16 - NAME_WIDTH = max([len(s) for s,d in linedata]) - if NAME_WIDTH < NAME_WIDTH_LOWER_BOUND: - NAME_WIDTH = NAME_WIDTH_LOWER_BOUND - else: - NAME_WIDTH = NAME_WIDTH_UPPER_BOUND - - DOC_WIDTH = WIDTH - NAME_WIDTH - SPACING - for namestr, doc in linedata: - line = indent + namestr - if len(namestr) <= NAME_WIDTH: - line += ' ' * (NAME_WIDTH + SPACING - len(namestr)) - else: - lines.append(line) - line = indent + ' ' * (NAME_WIDTH + SPACING) - line += _summarize_doc(doc, DOC_WIDTH) - lines.append(line.rstrip()) - return lines - -def _summarize_doc(doc, length=60): - r"""Parse out a short one line summary from the given doclines. - - "doc" is the doc string to summarize. - "length" is the max length for the summary - - >>> _summarize_doc("this function does this") - 'this function does this' - >>> _summarize_doc("this function does this", 10) - 'this fu...' - >>> _summarize_doc("this function does this\nand that") - 'this function does this and that' - >>> _summarize_doc("this function does this\n\nand that") - 'this function does this' - """ - import re - if doc is None: - return "" - assert length > 3, "length <= 3 is absurdly short for a doc summary" - doclines = doc.strip().splitlines(0) - if not doclines: - return "" - - summlines = [] - for i, line in enumerate(doclines): - stripped = line.strip() - if not stripped: - break - summlines.append(stripped) - if len(''.join(summlines)) >= length: - break - - summary = ' '.join(summlines) - if len(summary) > length: - summary = summary[:length-3] + "..." - return summary - - -def line2argv(line): - r"""Parse the given line into an argument vector. - - "line" is the line of input to parse. - - This may get niggly when dealing with quoting and escaping. The - current state of this parsing may not be completely thorough/correct - in this respect. - - >>> from cmdln import line2argv - >>> line2argv("foo") - ['foo'] - >>> line2argv("foo bar") - ['foo', 'bar'] - >>> line2argv("foo bar ") - ['foo', 'bar'] - >>> line2argv(" foo bar") - ['foo', 'bar'] - - Quote handling: - - >>> line2argv("'foo bar'") - ['foo bar'] - >>> line2argv('"foo bar"') - ['foo bar'] - >>> line2argv(r'"foo\"bar"') - ['foo"bar'] - >>> line2argv("'foo bar' spam") - ['foo bar', 'spam'] - >>> line2argv("'foo 'bar spam") - ['foo bar', 'spam'] - - >>> line2argv('some\tsimple\ttests') - ['some', 'simple', 'tests'] - >>> line2argv('a "more complex" test') - ['a', 'more complex', 'test'] - >>> line2argv('a more="complex test of " quotes') - ['a', 'more=complex test of ', 'quotes'] - >>> line2argv('a more" complex test of " quotes') - ['a', 'more complex test of ', 'quotes'] - >>> line2argv('an "embedded \\"quote\\""') - ['an', 'embedded "quote"'] - - # Komodo bug 48027 - >>> line2argv('foo bar C:\\') - ['foo', 'bar', 'C:\\'] - - # Komodo change 127581 - >>> line2argv(r'"\test\slash" "foo bar" "foo\"bar"') - ['\\test\\slash', 'foo bar', 'foo"bar'] - - # Komodo change 127629 - >>> if sys.platform == "win32": - ... line2argv(r'\foo\bar') == ['\\foo\\bar'] - ... line2argv(r'\\foo\\bar') == ['\\\\foo\\\\bar'] - ... line2argv('"foo') == ['foo'] - ... else: - ... line2argv(r'\foo\bar') == ['foobar'] - ... line2argv(r'\\foo\\bar') == ['\\foo\\bar'] - ... try: - ... line2argv('"foo') - ... except ValueError, ex: - ... "not terminated" in str(ex) - True - True - True - """ - import string - line = line.strip() - argv = [] - state = "default" - arg = None # the current argument being parsed - i = -1 - while 1: - i += 1 - if i >= len(line): break - ch = line[i] - - if ch == "\\" and i+1 < len(line): - # escaped char always added to arg, regardless of state - if arg is None: arg = "" - if (sys.platform == "win32" - or state in ("double-quoted", "single-quoted") - ) and line[i+1] not in tuple('"\''): - arg += ch - i += 1 - arg += line[i] - continue - - if state == "single-quoted": - if ch == "'": - state = "default" - else: - arg += ch - elif state == "double-quoted": - if ch == '"': - state = "default" - else: - arg += ch - elif state == "default": - if ch == '"': - if arg is None: arg = "" - state = "double-quoted" - elif ch == "'": - if arg is None: arg = "" - state = "single-quoted" - elif ch in string.whitespace: - if arg is not None: - argv.append(arg) - arg = None - else: - if arg is None: arg = "" - arg += ch - if arg is not None: - argv.append(arg) - if not sys.platform == "win32" and state != "default": - raise ValueError("command line is not terminated: unfinished %s " - "segment" % state) - return argv - - -def argv2line(argv): - r"""Put together the given argument vector into a command line. - - "argv" is the argument vector to process. - - >>> from cmdln import argv2line - >>> argv2line(['foo']) - 'foo' - >>> argv2line(['foo', 'bar']) - 'foo bar' - >>> argv2line(['foo', 'bar baz']) - 'foo "bar baz"' - >>> argv2line(['foo"bar']) - 'foo"bar' - >>> print argv2line(['foo" bar']) - 'foo" bar' - >>> print argv2line(["foo' bar"]) - "foo' bar" - >>> argv2line(["foo'bar"]) - "foo'bar" - """ - escapedArgs = [] - for arg in argv: - if ' ' in arg and '"' not in arg: - arg = '"'+arg+'"' - elif ' ' in arg and "'" not in arg: - arg = "'"+arg+"'" - elif ' ' in arg: - arg = arg.replace('"', r'\"') - arg = '"'+arg+'"' - escapedArgs.append(arg) - return ' '.join(escapedArgs) - - -# Recipe: dedent (0.1) in /Users/trentm/tm/recipes/cookbook -def _dedentlines(lines, tabsize=8, skip_first_line=False): - """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines - - "lines" is a list of lines to dedent. - "tabsize" is the tab width to use for indent width calculations. - "skip_first_line" is a boolean indicating if the first line should - be skipped for calculating the indent width and for dedenting. - This is sometimes useful for docstrings and similar. - - Same as dedent() except operates on a sequence of lines. Note: the - lines list is modified **in-place**. - """ - DEBUG = False - if DEBUG: - print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\ - % (tabsize, skip_first_line) - indents = [] - margin = None - for i, line in enumerate(lines): - if i == 0 and skip_first_line: continue - indent = 0 - for ch in line: - if ch == ' ': - indent += 1 - elif ch == '\t': - indent += tabsize - (indent % tabsize) - elif ch in '\r\n': - continue # skip all-whitespace lines - else: - break - else: - continue # skip all-whitespace lines - if DEBUG: print "dedent: indent=%d: %r" % (indent, line) - if margin is None: - margin = indent - else: - margin = min(margin, indent) - if DEBUG: print "dedent: margin=%r" % margin - - if margin is not None and margin > 0: - for i, line in enumerate(lines): - if i == 0 and skip_first_line: continue - removed = 0 - for j, ch in enumerate(line): - if ch == ' ': - removed += 1 - elif ch == '\t': - removed += tabsize - (removed % tabsize) - elif ch in '\r\n': - if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line - lines[i] = lines[i][j:] - break - else: - raise ValueError("unexpected non-whitespace char %r in " - "line %r while removing %d-space margin" - % (ch, line, margin)) - if DEBUG: - print "dedent: %r: %r -> removed %d/%d"\ - % (line, ch, removed, margin) - if removed == margin: - lines[i] = lines[i][j+1:] - break - elif removed > margin: - lines[i] = ' '*(removed-margin) + lines[i][j+1:] - break - return lines - -def _dedent(text, tabsize=8, skip_first_line=False): - """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text - - "text" is the text to dedent. - "tabsize" is the tab width to use for indent width calculations. - "skip_first_line" is a boolean indicating if the first line should - be skipped for calculating the indent width and for dedenting. - This is sometimes useful for docstrings and similar. - - textwrap.dedent(s), but don't expand tabs to spaces - """ - lines = text.splitlines(1) - _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line) - return ''.join(lines) - - -def _get_indent(marker, s, tab_width=8): - """_get_indent(marker, s, tab_width=8) -> - (<indentation-of-'marker'>, <indentation-width>)""" - # Figure out how much the marker is indented. - INDENT_CHARS = tuple(' \t') - start = s.index(marker) - i = start - while i > 0: - if s[i-1] not in INDENT_CHARS: - break - i -= 1 - indent = s[i:start] - indent_width = 0 - for ch in indent: - if ch == ' ': - indent_width += 1 - elif ch == '\t': - indent_width += tab_width - (indent_width % tab_width) - return indent, indent_width - -def _get_trailing_whitespace(marker, s): - """Return the whitespace content trailing the given 'marker' in string 's', - up to and including a newline. - """ - suffix = '' - start = s.index(marker) + len(marker) - i = start - while i < len(s): - if s[i] in ' \t': - suffix += s[i] - elif s[i] in '\r\n': - suffix += s[i] - if s[i] == '\r' and i+1 < len(s) and s[i+1] == '\n': - suffix += s[i+1] - break - else: - break - i += 1 - return suffix - - - -#---- bash completion support -# Note: This is still experimental. I expect to change this -# significantly. -# -# To get Bash completion for a cmdln.Cmdln class, run the following -# bash command: -# $ complete -C 'python -m cmdln /path/to/script.py CmdlnClass' cmdname -# For example: -# $ complete -C 'python -m cmdln ~/bin/svn.py SVN' svn -# -#TODO: Simplify the above so don't have to given path to script (try to -# find it on PATH, if possible). Could also make class name -# optional if there is only one in the module (common case). - -if __name__ == "__main__" and len(sys.argv) == 6: - def _log(s): - return # no-op, comment out for debugging - from os.path import expanduser - fout = open(expanduser("~/tmp/bashcpln.log"), 'a') - fout.write(str(s) + '\n') - fout.close() - - # Recipe: module_from_path (1.0.1+) - def _module_from_path(path): - import imp, os, sys - path = os.path.expanduser(path) - dir = os.path.dirname(path) or os.curdir - name = os.path.splitext(os.path.basename(path))[0] - sys.path.insert(0, dir) - try: - iinfo = imp.find_module(name, [dir]) - return imp.load_module(name, *iinfo) - finally: - sys.path.remove(dir) - - def _get_bash_cplns(script_path, class_name, cmd_name, - token, preceding_token): - _log('--') - _log('get_cplns(%r, %r, %r, %r, %r)' - % (script_path, class_name, cmd_name, token, preceding_token)) - comp_line = os.environ["COMP_LINE"] - comp_point = int(os.environ["COMP_POINT"]) - _log("COMP_LINE: %r" % comp_line) - _log("COMP_POINT: %r" % comp_point) - - try: - script = _module_from_path(script_path) - except ImportError, ex: - _log("error importing `%s': %s" % (script_path, ex)) - return [] - shell = getattr(script, class_name)() - cmd_map = shell._get_canonical_map() - del cmd_map["EOF"] - - # Determine if completing the sub-command name. - parts = comp_line[:comp_point].split(None, 1) - _log(parts) - if len(parts) == 1 or not (' ' in parts[1] or '\t' in parts[1]): - #TODO: if parts[1].startswith('-'): handle top-level opts - _log("complete sub-command names") - matches = {} - for name, canon_name in cmd_map.items(): - if name.startswith(token): - matches[name] = canon_name - if not matches: - return [] - elif len(matches) == 1: - return matches.keys() - elif len(set(matches.values())) == 1: - return [matches.values()[0]] - else: - return matches.keys() - - # Otherwise, complete options for the given sub-command. - #TODO: refine this so it does the right thing with option args - if token.startswith('-'): - cmd_name = comp_line.split(None, 2)[1] - try: - cmd_canon_name = cmd_map[cmd_name] - except KeyError: - return [] - handler = shell._get_cmd_handler(cmd_canon_name) - optparser = getattr(handler, "optparser", None) - if optparser is None: - optparser = SubCmdOptionParser() - opt_strs = [] - for option in optparser.option_list: - for opt_str in option._short_opts + option._long_opts: - if opt_str.startswith(token): - opt_strs.append(opt_str) - return opt_strs - - return [] - - for cpln in _get_bash_cplns(*sys.argv[1:]): - print cpln - diff --git a/scripts/lib/mic/utils/errors.py b/scripts/lib/mic/utils/errors.py deleted file mode 100644 index 8d720f9080..0000000000 --- a/scripts/lib/mic/utils/errors.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2007 Red Hat, Inc. -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -class CreatorError(Exception): - """An exception base class for all imgcreate errors.""" - keyword = '<creator>' - - def __init__(self, msg): - self.msg = msg - - def __str__(self): - if isinstance(self.msg, unicode): - self.msg = self.msg.encode('utf-8', 'ignore') - else: - self.msg = str(self.msg) - return self.keyword + self.msg - -class Usage(CreatorError): - keyword = '<usage>' - - def __str__(self): - if isinstance(self.msg, unicode): - self.msg = self.msg.encode('utf-8', 'ignore') - else: - self.msg = str(self.msg) - return self.keyword + self.msg + ', please use "--help" for more info' - -class Abort(CreatorError): - keyword = '' - -class ConfigError(CreatorError): - keyword = '<config>' - -class KsError(CreatorError): - keyword = '<kickstart>' - -class RepoError(CreatorError): - keyword = '<repo>' - -class RpmError(CreatorError): - keyword = '<rpm>' - -class MountError(CreatorError): - keyword = '<mount>' - -class SnapshotError(CreatorError): - keyword = '<snapshot>' - -class SquashfsError(CreatorError): - keyword = '<squashfs>' - -class BootstrapError(CreatorError): - keyword = '<bootstrap>' - -class RuntimeError(CreatorError): - keyword = '<runtime>' diff --git a/scripts/lib/mic/utils/fs_related.py b/scripts/lib/mic/utils/fs_related.py deleted file mode 100644 index dd420e88dc..0000000000 --- a/scripts/lib/mic/utils/fs_related.py +++ /dev/null @@ -1,1060 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2007, Red Hat, Inc. -# Copyright (c) 2009, 2010, 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -from __future__ import with_statement -import os -import sys -import errno -import stat -import random -import string -import time -import uuid - -from mic import msger -from mic.utils import runner -from mic.utils.errors import * -from mic.utils.oe.misc import * - -def find_binary_inchroot(binary, chroot): - paths = ["/usr/sbin", - "/usr/bin", - "/sbin", - "/bin" - ] - - for path in paths: - bin_path = "%s/%s" % (path, binary) - if os.path.exists("%s/%s" % (chroot, bin_path)): - return bin_path - return None - -def find_binary_path(binary): - if os.environ.has_key("PATH"): - paths = os.environ["PATH"].split(":") - else: - paths = [] - if os.environ.has_key("HOME"): - paths += [os.environ["HOME"] + "/bin"] - paths += ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"] - - for path in paths: - bin_path = "%s/%s" % (path, binary) - if os.path.exists(bin_path): - return bin_path - - print "External command '%s' not found, exiting." % binary - print " (Please install '%s' on your host system)" % binary - sys.exit(1) - -def makedirs(dirname): - """A version of os.makedirs() that doesn't throw an - exception if the leaf directory already exists. - """ - try: - os.makedirs(dirname) - except OSError, err: - if err.errno != errno.EEXIST: - raise - -def mksquashfs(in_img, out_img): - fullpathmksquashfs = find_binary_path("mksquashfs") - args = [fullpathmksquashfs, in_img, out_img] - - if not sys.stdout.isatty(): - args.append("-no-progress") - - ret = runner.show(args) - if ret != 0: - raise SquashfsError("'%s' exited with error (%d)" % (' '.join(args), ret)) - -def resize2fs(fs, size): - resize2fs = find_binary_path("resize2fs") - if size == 0: - # it means to minimalize it - return runner.show([resize2fs, '-M', fs]) - else: - return runner.show([resize2fs, fs, "%sK" % (size / 1024,)]) - -def my_fuser(fp): - fuser = find_binary_path("fuser") - if not os.path.exists(fp): - return False - - rc = runner.quiet([fuser, "-s", fp]) - if rc == 0: - for pid in runner.outs([fuser, fp]).split(): - fd = open("/proc/%s/cmdline" % pid, "r") - cmdline = fd.read() - fd.close() - if cmdline[:-1] == "/bin/bash": - return True - - # not found - return False - -class BindChrootMount: - """Represents a bind mount of a directory into a chroot.""" - def __init__(self, src, chroot, dest = None, option = None): - self.root = os.path.abspath(os.path.expanduser(chroot)) - self.option = option - - self.orig_src = self.src = src - if os.path.islink(src): - self.src = os.readlink(src) - if not self.src.startswith('/'): - self.src = os.path.abspath(os.path.join(os.path.dirname(src), - self.src)) - - if not dest: - dest = self.src - self.dest = os.path.join(self.root, dest.lstrip('/')) - - self.mounted = False - self.mountcmd = find_binary_path("mount") - self.umountcmd = find_binary_path("umount") - - def ismounted(self): - with open('/proc/mounts') as f: - for line in f: - if line.split()[1] == os.path.abspath(self.dest): - return True - - return False - - def has_chroot_instance(self): - lock = os.path.join(self.root, ".chroot.lock") - return my_fuser(lock) - - def mount(self): - if self.mounted or self.ismounted(): - return - - makedirs(self.dest) - rc = runner.show([self.mountcmd, "--bind", self.src, self.dest]) - if rc != 0: - raise MountError("Bind-mounting '%s' to '%s' failed" % - (self.src, self.dest)) - if self.option: - rc = runner.show([self.mountcmd, "--bind", "-o", "remount,%s" % self.option, self.dest]) - if rc != 0: - raise MountError("Bind-remounting '%s' failed" % self.dest) - - self.mounted = True - if os.path.islink(self.orig_src): - dest = os.path.join(self.root, self.orig_src.lstrip('/')) - if not os.path.exists(dest): - os.symlink(self.src, dest) - - def unmount(self): - if self.has_chroot_instance(): - return - - if self.ismounted(): - runner.show([self.umountcmd, "-l", self.dest]) - self.mounted = False - -class LoopbackMount: - """LoopbackMount compatibility layer for old API""" - def __init__(self, lofile, mountdir, fstype = None): - self.diskmount = DiskMount(LoopbackDisk(lofile,size = 0),mountdir,fstype,rmmountdir = True) - self.losetup = False - self.losetupcmd = find_binary_path("losetup") - - def cleanup(self): - self.diskmount.cleanup() - - def unmount(self): - self.diskmount.unmount() - - def lounsetup(self): - if self.losetup: - runner.show([self.losetupcmd, "-d", self.loopdev]) - self.losetup = False - self.loopdev = None - - def loopsetup(self): - if self.losetup: - return - - self.loopdev = get_loop_device(self.losetupcmd, self.lofile) - self.losetup = True - - def mount(self): - self.diskmount.mount() - -class SparseLoopbackMount(LoopbackMount): - """SparseLoopbackMount compatibility layer for old API""" - def __init__(self, lofile, mountdir, size, fstype = None): - self.diskmount = DiskMount(SparseLoopbackDisk(lofile,size),mountdir,fstype,rmmountdir = True) - - def expand(self, create = False, size = None): - self.diskmount.disk.expand(create, size) - - def truncate(self, size = None): - self.diskmount.disk.truncate(size) - - def create(self): - self.diskmount.disk.create() - -class SparseExtLoopbackMount(SparseLoopbackMount): - """SparseExtLoopbackMount compatibility layer for old API""" - def __init__(self, lofile, mountdir, size, fstype, blocksize, fslabel): - self.diskmount = ExtDiskMount(SparseLoopbackDisk(lofile,size), mountdir, fstype, blocksize, fslabel, rmmountdir = True) - - - def __format_filesystem(self): - self.diskmount.__format_filesystem() - - def create(self): - self.diskmount.disk.create() - - def resize(self, size = None): - return self.diskmount.__resize_filesystem(size) - - def mount(self): - self.diskmount.mount() - - def __fsck(self): - self.extdiskmount.__fsck() - - def __get_size_from_filesystem(self): - return self.diskmount.__get_size_from_filesystem() - - def __resize_to_minimal(self): - return self.diskmount.__resize_to_minimal() - - def resparse(self, size = None): - return self.diskmount.resparse(size) - -class Disk: - """Generic base object for a disk - - The 'create' method must make the disk visible as a block device - eg - by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup' - method must undo the 'create' operation. - """ - def __init__(self, size, device = None): - self._device = device - self._size = size - - def create(self): - pass - - def cleanup(self): - pass - - def get_device(self): - return self._device - def set_device(self, path): - self._device = path - device = property(get_device, set_device) - - def get_size(self): - return self._size - size = property(get_size) - - -class RawDisk(Disk): - """A Disk backed by a block device. - Note that create() is a no-op. - """ - def __init__(self, size, device): - Disk.__init__(self, size, device) - - def fixed(self): - return True - - def exists(self): - return True - - -class DiskImage(Disk): - """ - A Disk backed by a file. - """ - def __init__(self, image_file, size): - Disk.__init__(self, size) - self.image_file = image_file - - def exists(self): - return os.path.exists(self.image_file) - - def create(self): - if self.device is not None: - return - - blocks = self.size / 1024 - if self.size - blocks * 1024: - blocks += 1 - - # create disk image - dd_cmd = "dd if=/dev/zero of=%s bs=1024 seek=%d count=1" % \ - (self.image_file, blocks) - rc, out = exec_cmd(dd_cmd) - - self.device = self.image_file - - -class LoopbackDisk(Disk): - """A Disk backed by a file via the loop module.""" - def __init__(self, lofile, size): - Disk.__init__(self, size) - self.lofile = lofile - self.losetupcmd = find_binary_path("losetup") - - def fixed(self): - return False - - def exists(self): - return os.path.exists(self.lofile) - - def create(self): - if self.device is not None: - return - - self.device = get_loop_device(self.losetupcmd, self.lofile) - - def cleanup(self): - if self.device is None: - return - msger.debug("Losetup remove %s" % self.device) - rc = runner.show([self.losetupcmd, "-d", self.device]) - self.device = None - -class SparseLoopbackDisk(LoopbackDisk): - """A Disk backed by a sparse file via the loop module.""" - def __init__(self, lofile, size): - LoopbackDisk.__init__(self, lofile, size) - - def expand(self, create = False, size = None): - flags = os.O_WRONLY - if create: - flags |= os.O_CREAT - if not os.path.exists(self.lofile): - makedirs(os.path.dirname(self.lofile)) - - if size is None: - size = self.size - - msger.debug("Extending sparse file %s to %d" % (self.lofile, size)) - if create: - fd = os.open(self.lofile, flags, 0644) - else: - fd = os.open(self.lofile, flags) - - if size <= 0: - size = 1 - try: - os.ftruncate(fd, size) - except: - # may be limited by 2G in 32bit env - os.ftruncate(fd, 2**31L) - - os.close(fd) - - def truncate(self, size = None): - if size is None: - size = self.size - - msger.debug("Truncating sparse file %s to %d" % (self.lofile, size)) - fd = os.open(self.lofile, os.O_WRONLY) - os.ftruncate(fd, size) - os.close(fd) - - def create(self): - self.expand(create = True) - LoopbackDisk.create(self) - -class Mount: - """A generic base class to deal with mounting things.""" - def __init__(self, mountdir): - self.mountdir = mountdir - - def cleanup(self): - self.unmount() - - def mount(self, options = None): - pass - - def unmount(self): - pass - -class DiskMount(Mount): - """A Mount object that handles mounting of a Disk.""" - def __init__(self, disk, mountdir, fstype = None, rmmountdir = True): - Mount.__init__(self, mountdir) - - self.disk = disk - self.fstype = fstype - self.rmmountdir = rmmountdir - - self.mounted = False - self.rmdir = False - if fstype: - self.mkfscmd = find_binary_path("mkfs." + self.fstype) - else: - self.mkfscmd = None - self.mountcmd = find_binary_path("mount") - self.umountcmd = find_binary_path("umount") - - def cleanup(self): - Mount.cleanup(self) - self.disk.cleanup() - - def unmount(self): - if self.mounted: - msger.debug("Unmounting directory %s" % self.mountdir) - runner.quiet('sync') # sync the data on this mount point - rc = runner.show([self.umountcmd, "-l", self.mountdir]) - if rc == 0: - self.mounted = False - else: - raise MountError("Failed to umount %s" % self.mountdir) - if self.rmdir and not self.mounted: - try: - os.rmdir(self.mountdir) - except OSError, e: - pass - self.rmdir = False - - - def __create(self): - self.disk.create() - - - def mount(self, options = None): - if self.mounted: - return - - if not os.path.isdir(self.mountdir): - msger.debug("Creating mount point %s" % self.mountdir) - os.makedirs(self.mountdir) - self.rmdir = self.rmmountdir - - self.__create() - - msger.debug("Mounting %s at %s" % (self.disk.device, self.mountdir)) - if options: - args = [ self.mountcmd, "-o", options, self.disk.device, self.mountdir ] - else: - args = [ self.mountcmd, self.disk.device, self.mountdir ] - if self.fstype: - args.extend(["-t", self.fstype]) - - rc = runner.show(args) - if rc != 0: - raise MountError("Failed to mount '%s' to '%s' with command '%s'. Retval: %s" % - (self.disk.device, self.mountdir, " ".join(args), rc)) - - self.mounted = True - -class ExtDiskMount(DiskMount): - """A DiskMount object that is able to format/resize ext[23] filesystems.""" - def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None): - DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) - self.blocksize = blocksize - self.fslabel = fslabel.replace("/", "") - self.uuid = str(uuid.uuid4()) - self.skipformat = skipformat - self.fsopts = fsopts - self.extopts = None - self.dumpe2fs = find_binary_path("dumpe2fs") - self.tune2fs = find_binary_path("tune2fs") - - def __parse_field(self, output, field): - for line in output.split("\n"): - if line.startswith(field + ":"): - return line[len(field) + 1:].strip() - - raise KeyError("Failed to find field '%s' in output" % field) - - def __format_filesystem(self): - if self.skipformat: - msger.debug("Skip filesystem format.") - return - - msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device)) - cmdlist = [self.mkfscmd, "-F", "-L", self.fslabel, "-m", "1", "-b", - str(self.blocksize), "-U", self.uuid] - if self.extopts: - cmdlist.extend(self.extopts.split()) - cmdlist.extend([self.disk.device]) - - rc, errout = runner.runtool(cmdlist, catch=2) - if rc != 0: - raise MountError("Error creating %s filesystem on disk %s:\n%s" % - (self.fstype, self.disk.device, errout)) - - if not self.extopts: - msger.debug("Tuning filesystem on %s" % self.disk.device) - runner.show([self.tune2fs, "-c0", "-i0", "-Odir_index", "-ouser_xattr,acl", self.disk.device]) - - def __resize_filesystem(self, size = None): - current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] - - if size is None: - size = self.disk.size - - if size == current_size: - return - - if size > current_size: - self.disk.expand(size) - - self.__fsck() - - resize2fs(self.disk.lofile, size) - return size - - def __create(self): - resize = False - if not self.disk.fixed() and self.disk.exists(): - resize = True - - self.disk.create() - - if resize: - self.__resize_filesystem() - else: - self.__format_filesystem() - - def mount(self, options = None): - self.__create() - DiskMount.mount(self, options) - - def __fsck(self): - msger.info("Checking filesystem %s" % self.disk.lofile) - runner.quiet(["/sbin/e2fsck", "-f", "-y", self.disk.lofile]) - - def __get_size_from_filesystem(self): - return int(self.__parse_field(runner.outs([self.dumpe2fs, '-h', self.disk.lofile]), - "Block count")) * self.blocksize - - def __resize_to_minimal(self): - self.__fsck() - - # - # Use a binary search to find the minimal size - # we can resize the image to - # - bot = 0 - top = self.__get_size_from_filesystem() - while top != (bot + 1): - t = bot + ((top - bot) / 2) - - if not resize2fs(self.disk.lofile, t): - top = t - else: - bot = t - return top - - def resparse(self, size = None): - self.cleanup() - if size == 0: - minsize = 0 - else: - minsize = self.__resize_to_minimal() - self.disk.truncate(minsize) - - self.__resize_filesystem(size) - return minsize - -class VfatDiskMount(DiskMount): - """A DiskMount object that is able to format vfat/msdos filesystems.""" - def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None): - DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) - self.blocksize = blocksize - self.fslabel = fslabel.replace("/", "") - rand1 = random.randint(0, 2**16 - 1) - rand2 = random.randint(0, 2**16 - 1) - self.uuid = "%04X-%04X" % (rand1, rand2) - self.skipformat = skipformat - self.fsopts = fsopts - self.fsckcmd = find_binary_path("fsck." + self.fstype) - - def __format_filesystem(self): - if self.skipformat: - msger.debug("Skip filesystem format.") - return - - msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device)) - rc = runner.show([self.mkfscmd, "-n", self.fslabel, - "-i", self.uuid.replace("-", ""), self.disk.device]) - if rc != 0: - raise MountError("Error creating %s filesystem on disk %s" % (self.fstype,self.disk.device)) - - msger.verbose("Tuning filesystem on %s" % self.disk.device) - - def __resize_filesystem(self, size = None): - current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] - - if size is None: - size = self.disk.size - - if size == current_size: - return - - if size > current_size: - self.disk.expand(size) - - self.__fsck() - - #resize2fs(self.disk.lofile, size) - return size - - def __create(self): - resize = False - if not self.disk.fixed() and self.disk.exists(): - resize = True - - self.disk.create() - - if resize: - self.__resize_filesystem() - else: - self.__format_filesystem() - - def mount(self, options = None): - self.__create() - DiskMount.mount(self, options) - - def __fsck(self): - msger.debug("Checking filesystem %s" % self.disk.lofile) - runner.show([self.fsckcmd, "-y", self.disk.lofile]) - - def __get_size_from_filesystem(self): - return self.disk.size - - def __resize_to_minimal(self): - self.__fsck() - - # - # Use a binary search to find the minimal size - # we can resize the image to - # - bot = 0 - top = self.__get_size_from_filesystem() - return top - - def resparse(self, size = None): - self.cleanup() - minsize = self.__resize_to_minimal() - self.disk.truncate(minsize) - self.__resize_filesystem(size) - return minsize - -class BtrfsDiskMount(DiskMount): - """A DiskMount object that is able to format/resize btrfs filesystems.""" - def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None): - self.__check_btrfs() - DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) - self.blocksize = blocksize - self.fslabel = fslabel.replace("/", "") - self.uuid = None - self.skipformat = skipformat - self.fsopts = fsopts - self.blkidcmd = find_binary_path("blkid") - self.btrfsckcmd = find_binary_path("btrfsck") - - def __check_btrfs(self): - found = False - """ Need to load btrfs module to mount it """ - load_module("btrfs") - for line in open("/proc/filesystems").xreadlines(): - if line.find("btrfs") > -1: - found = True - break - if not found: - raise MountError("Your system can't mount btrfs filesystem, please make sure your kernel has btrfs support and the module btrfs.ko has been loaded.") - - # disable selinux, selinux will block write - if os.path.exists("/usr/sbin/setenforce"): - runner.show(["/usr/sbin/setenforce", "0"]) - - def __parse_field(self, output, field): - for line in output.split(" "): - if line.startswith(field + "="): - return line[len(field) + 1:].strip().replace("\"", "") - - raise KeyError("Failed to find field '%s' in output" % field) - - def __format_filesystem(self): - if self.skipformat: - msger.debug("Skip filesystem format.") - return - - msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device)) - rc = runner.show([self.mkfscmd, "-L", self.fslabel, self.disk.device]) - if rc != 0: - raise MountError("Error creating %s filesystem on disk %s" % (self.fstype,self.disk.device)) - - self.uuid = self.__parse_field(runner.outs([self.blkidcmd, self.disk.device]), "UUID") - - def __resize_filesystem(self, size = None): - current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] - - if size is None: - size = self.disk.size - - if size == current_size: - return - - if size > current_size: - self.disk.expand(size) - - self.__fsck() - return size - - def __create(self): - resize = False - if not self.disk.fixed() and self.disk.exists(): - resize = True - - self.disk.create() - - if resize: - self.__resize_filesystem() - else: - self.__format_filesystem() - - def mount(self, options = None): - self.__create() - DiskMount.mount(self, options) - - def __fsck(self): - msger.debug("Checking filesystem %s" % self.disk.lofile) - runner.quiet([self.btrfsckcmd, self.disk.lofile]) - - def __get_size_from_filesystem(self): - return self.disk.size - - def __resize_to_minimal(self): - self.__fsck() - - return self.__get_size_from_filesystem() - - def resparse(self, size = None): - self.cleanup() - minsize = self.__resize_to_minimal() - self.disk.truncate(minsize) - self.__resize_filesystem(size) - return minsize - -class DeviceMapperSnapshot(object): - def __init__(self, imgloop, cowloop): - self.imgloop = imgloop - self.cowloop = cowloop - - self.__created = False - self.__name = None - self.dmsetupcmd = find_binary_path("dmsetup") - - """Load dm_snapshot if it isn't loaded""" - load_module("dm_snapshot") - - def get_path(self): - if self.__name is None: - return None - return os.path.join("/dev/mapper", self.__name) - path = property(get_path) - - def create(self): - if self.__created: - return - - self.imgloop.create() - self.cowloop.create() - - self.__name = "imgcreate-%d-%d" % (os.getpid(), - random.randint(0, 2**16)) - - size = os.stat(self.imgloop.lofile)[stat.ST_SIZE] - - table = "0 %d snapshot %s %s p 8" % (size / 512, - self.imgloop.device, - self.cowloop.device) - - args = [self.dmsetupcmd, "create", self.__name, "--table", table] - if runner.show(args) != 0: - self.cowloop.cleanup() - self.imgloop.cleanup() - raise SnapshotError("Could not create snapshot device using: " + ' '.join(args)) - - self.__created = True - - def remove(self, ignore_errors = False): - if not self.__created: - return - - time.sleep(2) - rc = runner.show([self.dmsetupcmd, "remove", self.__name]) - if not ignore_errors and rc != 0: - raise SnapshotError("Could not remove snapshot device") - - self.__name = None - self.__created = False - - self.cowloop.cleanup() - self.imgloop.cleanup() - - def get_cow_used(self): - if not self.__created: - return 0 - - # - # dmsetup status on a snapshot returns e.g. - # "0 8388608 snapshot 416/1048576" - # or, more generally: - # "A B snapshot C/D" - # where C is the number of 512 byte sectors in use - # - out = runner.outs([self.dmsetupcmd, "status", self.__name]) - try: - return int((out.split()[3]).split('/')[0]) * 512 - except ValueError: - raise SnapshotError("Failed to parse dmsetup status: " + out) - -def create_image_minimizer(path, image, minimal_size): - """ - Builds a copy-on-write image which can be used to - create a device-mapper snapshot of an image where - the image's filesystem is as small as possible - - The steps taken are: - 1) Create a sparse COW - 2) Loopback mount the image and the COW - 3) Create a device-mapper snapshot of the image - using the COW - 4) Resize the filesystem to the minimal size - 5) Determine the amount of space used in the COW - 6) Restroy the device-mapper snapshot - 7) Truncate the COW, removing unused space - 8) Create a squashfs of the COW - """ - imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter - - cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"), - 64L * 1024L * 1024L) - - snapshot = DeviceMapperSnapshot(imgloop, cowloop) - - try: - snapshot.create() - - resize2fs(snapshot.path, minimal_size) - - cow_used = snapshot.get_cow_used() - finally: - snapshot.remove(ignore_errors = (not sys.exc_info()[0] is None)) - - cowloop.truncate(cow_used) - - mksquashfs(cowloop.lofile, path) - - os.unlink(cowloop.lofile) - -def load_module(module): - found = False - for line in open('/proc/modules').xreadlines(): - if line.startswith("%s " % module): - found = True - break - if not found: - msger.info("Loading %s..." % module) - runner.quiet(['modprobe', module]) - -class LoopDevice(object): - def __init__(self, loopid=None): - self.device = None - self.loopid = loopid - self.created = False - self.kpartxcmd = find_binary_path("kpartx") - self.losetupcmd = find_binary_path("losetup") - - def register(self, device): - self.device = device - self.loopid = None - self.created = True - - def reg_atexit(self): - import atexit - atexit.register(self.close) - - def _genloopid(self): - import glob - if not glob.glob("/dev/loop[0-9]*"): - return 10 - - fint = lambda x: x[9:].isdigit() and int(x[9:]) or 0 - maxid = 1 + max(filter(lambda x: x<100, - map(fint, glob.glob("/dev/loop[0-9]*")))) - if maxid < 10: maxid = 10 - if maxid >= 100: raise - return maxid - - def _kpseek(self, device): - rc, out = runner.runtool([self.kpartxcmd, '-l', '-v', device]) - if rc != 0: - raise MountError("Can't query dm snapshot on %s" % device) - for line in out.splitlines(): - if line and line.startswith("loop"): - return True - return False - - def _loseek(self, device): - import re - rc, out = runner.runtool([self.losetupcmd, '-a']) - if rc != 0: - raise MountError("Failed to run 'losetup -a'") - for line in out.splitlines(): - m = re.match("([^:]+): .*", line) - if m and m.group(1) == device: - return True - return False - - def create(self): - if not self.created: - if not self.loopid: - self.loopid = self._genloopid() - self.device = "/dev/loop%d" % self.loopid - if os.path.exists(self.device): - if self._loseek(self.device): - raise MountError("Device busy: %s" % self.device) - else: - self.created = True - return - - mknod = find_binary_path('mknod') - rc = runner.show([mknod, '-m664', self.device, 'b', '7', str(self.loopid)]) - if rc != 0: - raise MountError("Failed to create device %s" % self.device) - else: - self.created = True - - def close(self): - if self.created: - try: - self.cleanup() - self.device = None - except MountError, e: - msger.error("%s" % e) - - def cleanup(self): - - if self.device is None: - return - - - if self._kpseek(self.device): - if self.created: - for i in range(3, os.sysconf("SC_OPEN_MAX")): - try: - os.close(i) - except: - pass - runner.quiet([self.kpartxcmd, "-d", self.device]) - if self._loseek(self.device): - runner.quiet([self.losetupcmd, "-d", self.device]) - # FIXME: should sleep a while between two loseek - if self._loseek(self.device): - msger.warning("Can't cleanup loop device %s" % self.device) - elif self.loopid: - os.unlink(self.device) - -DEVICE_PIDFILE_DIR = "/var/tmp/mic/device" -DEVICE_LOCKFILE = "/var/lock/__mic_loopdev.lock" - -def get_loop_device(losetupcmd, lofile): - global DEVICE_PIDFILE_DIR - global DEVICE_LOCKFILE - - import fcntl - makedirs(os.path.dirname(DEVICE_LOCKFILE)) - fp = open(DEVICE_LOCKFILE, 'w') - fcntl.flock(fp, fcntl.LOCK_EX) - try: - loopdev = None - devinst = LoopDevice() - - # clean up left loop device first - clean_loop_devices() - - # provide an avaible loop device - rc, out = runner.runtool([losetupcmd, "--find"]) - if rc == 0: - loopdev = out.split()[0] - devinst.register(loopdev) - if not loopdev or not os.path.exists(loopdev): - devinst.create() - loopdev = devinst.device - - # setup a loop device for image file - rc = runner.show([losetupcmd, loopdev, lofile]) - if rc != 0: - raise MountError("Failed to setup loop device for '%s'" % lofile) - - devinst.reg_atexit() - - # try to save device and pid - makedirs(DEVICE_PIDFILE_DIR) - pidfile = os.path.join(DEVICE_PIDFILE_DIR, os.path.basename(loopdev)) - if os.path.exists(pidfile): - os.unlink(pidfile) - with open(pidfile, 'w') as wf: - wf.write(str(os.getpid())) - - except MountError, err: - raise CreatorError("%s" % str(err)) - except: - raise - finally: - try: - fcntl.flock(fp, fcntl.LOCK_UN) - fp.close() - os.unlink(DEVICE_LOCKFILE) - except: - pass - - return loopdev - -def clean_loop_devices(piddir=DEVICE_PIDFILE_DIR): - if not os.path.exists(piddir) or not os.path.isdir(piddir): - return - - for loopdev in os.listdir(piddir): - pidfile = os.path.join(piddir, loopdev) - try: - with open(pidfile, 'r') as rf: - devpid = int(rf.read()) - except: - devpid = None - - # if the process using this device is alive, skip it - if not devpid or os.path.exists(os.path.join('/proc', str(devpid))): - continue - - # try to clean it up - try: - devinst = LoopDevice() - devinst.register(os.path.join('/dev', loopdev)) - devinst.cleanup() - os.unlink(pidfile) - except: - pass - diff --git a/scripts/lib/mic/utils/gpt_parser.py b/scripts/lib/mic/utils/gpt_parser.py deleted file mode 100644 index 5d43b70778..0000000000 --- a/scripts/lib/mic/utils/gpt_parser.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2013 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -""" This module implements a simple GPT partitions parser which can read the -GPT header and the GPT partition table. """ - -import struct -import uuid -import binascii -from mic.utils.errors import MountError - -_GPT_HEADER_FORMAT = "<8s4sIIIQQQQ16sQIII" -_GPT_HEADER_SIZE = struct.calcsize(_GPT_HEADER_FORMAT) -_GPT_ENTRY_FORMAT = "<16s16sQQQ72s" -_GPT_ENTRY_SIZE = struct.calcsize(_GPT_ENTRY_FORMAT) -_SUPPORTED_GPT_REVISION = '\x00\x00\x01\x00' - -def _stringify_uuid(binary_uuid): - """ A small helper function to transform a binary UUID into a string - format. """ - - uuid_str = str(uuid.UUID(bytes_le = binary_uuid)) - - return uuid_str.upper() - -def _calc_header_crc(raw_hdr): - """ Calculate GPT header CRC32 checksum. The 'raw_hdr' parameter has to - be a list or a tuple containing all the elements of the GPT header in a - "raw" form, meaning that it should simply contain "unpacked" disk data. - """ - - raw_hdr = list(raw_hdr) - raw_hdr[3] = 0 - raw_hdr = struct.pack(_GPT_HEADER_FORMAT, *raw_hdr) - - return binascii.crc32(raw_hdr) & 0xFFFFFFFF - -def _validate_header(raw_hdr): - """ Validate the GPT header. The 'raw_hdr' parameter has to be a list or a - tuple containing all the elements of the GPT header in a "raw" form, - meaning that it should simply contain "unpacked" disk data. """ - - # Validate the signature - if raw_hdr[0] != 'EFI PART': - raise MountError("GPT partition table not found") - - # Validate the revision - if raw_hdr[1] != _SUPPORTED_GPT_REVISION: - raise MountError("Unsupported GPT revision '%s', supported revision " \ - "is '%s'" % \ - (binascii.hexlify(raw_hdr[1]), - binascii.hexlify(_SUPPORTED_GPT_REVISION))) - - # Validate header size - if raw_hdr[2] != _GPT_HEADER_SIZE: - raise MountError("Bad GPT header size: %d bytes, expected %d" % \ - (raw_hdr[2], _GPT_HEADER_SIZE)) - - crc = _calc_header_crc(raw_hdr) - if raw_hdr[3] != crc: - raise MountError("GPT header crc mismatch: %#x, should be %#x" % \ - (crc, raw_hdr[3])) - -class GptParser: - """ GPT partition table parser. Allows reading the GPT header and the - partition table, as well as modifying the partition table records. """ - - def __init__(self, disk_path, sector_size = 512): - """ The class constructor which accepts the following parameters: - * disk_path - full path to the disk image or device node - * sector_size - size of a disk sector in bytes """ - - self.sector_size = sector_size - self.disk_path = disk_path - - try: - self._disk_obj = open(disk_path, 'r+b') - except IOError as err: - raise MountError("Cannot open file '%s' for reading GPT " \ - "partitions: %s" % (disk_path, err)) - - def __del__(self): - """ The class destructor. """ - - self._disk_obj.close() - - def _read_disk(self, offset, size): - """ A helper function which reads 'size' bytes from offset 'offset' of - the disk and checks all the error conditions. """ - - self._disk_obj.seek(offset) - try: - data = self._disk_obj.read(size) - except IOError as err: - raise MountError("cannot read from '%s': %s" % \ - (self.disk_path, err)) - - if len(data) != size: - raise MountError("cannot read %d bytes from offset '%d' of '%s', " \ - "read only %d bytes" % \ - (size, offset, self.disk_path, len(data))) - - return data - - def _write_disk(self, offset, buf): - """ A helper function which writes buffer 'buf' to offset 'offset' of - the disk. This function takes care of unaligned writes and checks all - the error conditions. """ - - # Since we may be dealing with a block device, we only can write in - # 'self.sector_size' chunks. Find the aligned starting and ending - # disk offsets to read. - start = (offset / self.sector_size) * self.sector_size - end = ((start + len(buf)) / self.sector_size + 1) * self.sector_size - - data = self._read_disk(start, end - start) - off = offset - start - data = data[:off] + buf + data[off + len(buf):] - - self._disk_obj.seek(start) - try: - self._disk_obj.write(data) - except IOError as err: - raise MountError("cannot write to '%s': %s" % (self.disk_path, err)) - - def read_header(self, primary = True): - """ Read and verify the GPT header and return a dictionary containing - the following elements: - - 'signature' : header signature - 'revision' : header revision - 'hdr_size' : header size in bytes - 'hdr_crc' : header CRC32 - 'hdr_lba' : LBA of this header - 'hdr_offs' : byte disk offset of this header - 'backup_lba' : backup header LBA - 'backup_offs' : byte disk offset of backup header - 'first_lba' : first usable LBA for partitions - 'first_offs' : first usable byte disk offset for partitions - 'last_lba' : last usable LBA for partitions - 'last_offs' : last usable byte disk offset for partitions - 'disk_uuid' : UUID of the disk - 'ptable_lba' : starting LBA of array of partition entries - 'ptable_offs' : disk byte offset of the start of the partition table - 'ptable_size' : partition table size in bytes - 'entries_cnt' : number of available partition table entries - 'entry_size' : size of a single partition entry - 'ptable_crc' : CRC32 of the partition table - 'primary' : a boolean, if 'True', this is the primary GPT header, - if 'False' - the secondary - 'primary_str' : contains string "primary" if this is the primary GPT - header, and "backup" otherwise - - This dictionary corresponds to the GPT header format. Please, see the - UEFI standard for the description of these fields. - - If the 'primary' parameter is 'True', the primary GPT header is read, - otherwise the backup GPT header is read instead. """ - - # Read and validate the primary GPT header - raw_hdr = self._read_disk(self.sector_size, _GPT_HEADER_SIZE) - raw_hdr = struct.unpack(_GPT_HEADER_FORMAT, raw_hdr) - _validate_header(raw_hdr) - primary_str = "primary" - - if not primary: - # Read and validate the backup GPT header - raw_hdr = self._read_disk(raw_hdr[6] * self.sector_size, _GPT_HEADER_SIZE) - raw_hdr = struct.unpack(_GPT_HEADER_FORMAT, raw_hdr) - _validate_header(raw_hdr) - primary_str = "backup" - - return { 'signature' : raw_hdr[0], - 'revision' : raw_hdr[1], - 'hdr_size' : raw_hdr[2], - 'hdr_crc' : raw_hdr[3], - 'hdr_lba' : raw_hdr[5], - 'hdr_offs' : raw_hdr[5] * self.sector_size, - 'backup_lba' : raw_hdr[6], - 'backup_offs' : raw_hdr[6] * self.sector_size, - 'first_lba' : raw_hdr[7], - 'first_offs' : raw_hdr[7] * self.sector_size, - 'last_lba' : raw_hdr[8], - 'last_offs' : raw_hdr[8] * self.sector_size, - 'disk_uuid' :_stringify_uuid(raw_hdr[9]), - 'ptable_lba' : raw_hdr[10], - 'ptable_offs' : raw_hdr[10] * self.sector_size, - 'ptable_size' : raw_hdr[11] * raw_hdr[12], - 'entries_cnt' : raw_hdr[11], - 'entry_size' : raw_hdr[12], - 'ptable_crc' : raw_hdr[13], - 'primary' : primary, - 'primary_str' : primary_str } - - def _read_raw_ptable(self, header): - """ Read and validate primary or backup partition table. The 'header' - argument is the GPT header. If it is the primary GPT header, then the - primary partition table is read and validated, otherwise - the backup - one. The 'header' argument is a dictionary which is returned by the - 'read_header()' method. """ - - raw_ptable = self._read_disk(header['ptable_offs'], - header['ptable_size']) - - crc = binascii.crc32(raw_ptable) & 0xFFFFFFFF - if crc != header['ptable_crc']: - raise MountError("Partition table at LBA %d (%s) is corrupted" % \ - (header['ptable_lba'], header['primary_str'])) - - return raw_ptable - - def get_partitions(self, primary = True): - """ This is a generator which parses the GPT partition table and - generates the following dictionary for each partition: - - 'index' : the index of the partition table endry - 'offs' : byte disk offset of the partition table entry - 'type_uuid' : partition type UUID - 'part_uuid' : partition UUID - 'first_lba' : the first LBA - 'last_lba' : the last LBA - 'flags' : attribute flags - 'name' : partition name - 'primary' : a boolean, if 'True', this is the primary partition - table, if 'False' - the secondary - 'primary_str' : contains string "primary" if this is the primary GPT - header, and "backup" otherwise - - This dictionary corresponds to the GPT header format. Please, see the - UEFI standard for the description of these fields. - - If the 'primary' parameter is 'True', partitions from the primary GPT - partition table are generated, otherwise partitions from the backup GPT - partition table are generated. """ - - if primary: - primary_str = "primary" - else: - primary_str = "backup" - - header = self.read_header(primary) - raw_ptable = self._read_raw_ptable(header) - - for index in xrange(0, header['entries_cnt']): - start = header['entry_size'] * index - end = start + header['entry_size'] - raw_entry = struct.unpack(_GPT_ENTRY_FORMAT, raw_ptable[start:end]) - - if raw_entry[2] == 0 or raw_entry[3] == 0: - continue - - part_name = str(raw_entry[5].decode('UTF-16').split('\0', 1)[0]) - - yield { 'index' : index, - 'offs' : header['ptable_offs'] + start, - 'type_uuid' : _stringify_uuid(raw_entry[0]), - 'part_uuid' : _stringify_uuid(raw_entry[1]), - 'first_lba' : raw_entry[2], - 'last_lba' : raw_entry[3], - 'flags' : raw_entry[4], - 'name' : part_name, - 'primary' : primary, - 'primary_str' : primary_str } - - def _change_partition(self, header, entry): - """ A helper function for 'change_partitions()' which changes a - a paricular instance of the partition table (primary or backup). """ - - if entry['index'] >= header['entries_cnt']: - raise MountError("Partition table at LBA %d has only %d " \ - "records cannot change record number %d" % \ - (header['entries_cnt'], entry['index'])) - # Read raw GPT header - raw_hdr = self._read_disk(header['hdr_offs'], _GPT_HEADER_SIZE) - raw_hdr = list(struct.unpack(_GPT_HEADER_FORMAT, raw_hdr)) - _validate_header(raw_hdr) - - # Prepare the new partition table entry - raw_entry = struct.pack(_GPT_ENTRY_FORMAT, - uuid.UUID(entry['type_uuid']).bytes_le, - uuid.UUID(entry['part_uuid']).bytes_le, - entry['first_lba'], - entry['last_lba'], - entry['flags'], - entry['name'].encode('UTF-16')) - - # Write the updated entry to the disk - entry_offs = header['ptable_offs'] + \ - header['entry_size'] * entry['index'] - self._write_disk(entry_offs, raw_entry) - - # Calculate and update partition table CRC32 - raw_ptable = self._read_disk(header['ptable_offs'], - header['ptable_size']) - raw_hdr[13] = binascii.crc32(raw_ptable) & 0xFFFFFFFF - - # Calculate and update the GPT header CRC - raw_hdr[3] = _calc_header_crc(raw_hdr) - - # Write the updated header to the disk - raw_hdr = struct.pack(_GPT_HEADER_FORMAT, *raw_hdr) - self._write_disk(header['hdr_offs'], raw_hdr) - - def change_partition(self, entry): - """ Change a GPT partition. The 'entry' argument has the same format as - 'get_partitions()' returns. This function simply changes the partition - table record corresponding to 'entry' in both, the primary and the - backup GPT partition tables. The parition table CRC is re-calculated - and the GPT headers are modified accordingly. """ - - # Change the primary partition table - header = self.read_header(True) - self._change_partition(header, entry) - - # Change the backup partition table - header = self.read_header(False) - self._change_partition(header, entry) diff --git a/scripts/lib/mic/utils/grabber.py b/scripts/lib/mic/utils/grabber.py deleted file mode 100644 index 45e30b4fb0..0000000000 --- a/scripts/lib/mic/utils/grabber.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/python - -import os -import sys -import rpm -import fcntl -import struct -import termios - -from mic import msger -from mic.utils import runner -from mic.utils.errors import CreatorError - -from urlgrabber import grabber -from urlgrabber import __version__ as grabber_version - -if rpm.labelCompare(grabber_version.split('.'), '3.9.0'.split('.')) == -1: - msger.warning("Version of python-urlgrabber is %s, lower than '3.9.0', " - "you may encounter some network issues" % grabber_version) - -def myurlgrab(url, filename, proxies, progress_obj = None): - g = grabber.URLGrabber() - if progress_obj is None: - progress_obj = TextProgress() - - if url.startswith("file:/"): - filepath = "/%s" % url.replace("file:", "").lstrip('/') - if not os.path.exists(filepath): - raise CreatorError("URLGrabber error: can't find file %s" % url) - if url.endswith('.rpm'): - return filepath - else: - # untouch repometadata in source path - runner.show(['cp', '-f', filepath, filename]) - - else: - try: - filename = g.urlgrab(url=str(url), - filename=filename, - ssl_verify_host=False, - ssl_verify_peer=False, - proxies=proxies, - http_headers=(('Pragma', 'no-cache'),), - quote=0, - progress_obj=progress_obj) - except grabber.URLGrabError, err: - msg = str(err) - if msg.find(url) < 0: - msg += ' on %s' % url - raise CreatorError(msg) - - return filename - -def terminal_width(fd=1): - """ Get the real terminal width """ - try: - buf = 'abcdefgh' - buf = fcntl.ioctl(fd, termios.TIOCGWINSZ, buf) - return struct.unpack('hhhh', buf)[1] - except: # IOError - return 80 - -def truncate_url(url, width): - return os.path.basename(url)[0:width] - -class TextProgress(object): - # make the class as singleton - _instance = None - def __new__(cls, *args, **kwargs): - if not cls._instance: - cls._instance = super(TextProgress, cls).__new__(cls, *args, **kwargs) - - return cls._instance - - def __init__(self, totalnum = None): - self.total = totalnum - self.counter = 1 - - def start(self, filename, url, *args, **kwargs): - self.url = url - self.termwidth = terminal_width() - msger.info("\r%-*s" % (self.termwidth, " ")) - if self.total is None: - msger.info("\rRetrieving %s ..." % truncate_url(self.url, self.termwidth - 15)) - else: - msger.info("\rRetrieving %s [%d/%d] ..." % (truncate_url(self.url, self.termwidth - 25), self.counter, self.total)) - - def update(self, *args): - pass - - def end(self, *args): - if self.counter == self.total: - msger.raw("\n") - - if self.total is not None: - self.counter += 1 - diff --git a/scripts/lib/mic/utils/misc.py b/scripts/lib/mic/utils/misc.py deleted file mode 100644 index 95241d7f15..0000000000 --- a/scripts/lib/mic/utils/misc.py +++ /dev/null @@ -1,1065 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2010, 2011 Intel Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import sys -import time -import tempfile -import re -import shutil -import glob -import hashlib -import subprocess -import platform -import traceback - - -try: - import sqlite3 as sqlite -except ImportError: - import sqlite - -try: - from xml.etree import cElementTree -except ImportError: - import cElementTree -xmlparse = cElementTree.parse - -from mic import msger -from mic.utils.errors import CreatorError, SquashfsError -from mic.utils.fs_related import find_binary_path, makedirs -from mic.utils.proxy import get_proxy_for -from mic.utils import runner - - -RPM_RE = re.compile("(.*)\.(.*) (.*)-(.*)") -RPM_FMT = "%(name)s.%(arch)s %(version)s-%(release)s" -SRPM_RE = re.compile("(.*)-(\d+.*)-(\d+\.\d+).src.rpm") - - -def build_name(kscfg, release=None, prefix = None, suffix = None): - """Construct and return an image name string. - - This is a utility function to help create sensible name and fslabel - strings. The name is constructed using the sans-prefix-and-extension - kickstart filename and the supplied prefix and suffix. - - kscfg -- a path to a kickstart file - release -- a replacement to suffix for image release - prefix -- a prefix to prepend to the name; defaults to None, which causes - no prefix to be used - suffix -- a suffix to append to the name; defaults to None, which causes - a YYYYMMDDHHMM suffix to be used - - Note, if maxlen is less then the len(suffix), you get to keep both pieces. - - """ - name = os.path.basename(kscfg) - idx = name.rfind('.') - if idx >= 0: - name = name[:idx] - - if release is not None: - suffix = "" - if prefix is None: - prefix = "" - if suffix is None: - suffix = time.strftime("%Y%m%d%H%M") - - if name.startswith(prefix): - name = name[len(prefix):] - - prefix = "%s-" % prefix if prefix else "" - suffix = "-%s" % suffix if suffix else "" - - ret = prefix + name + suffix - return ret - -def get_distro(): - """Detect linux distribution, support "meego" - """ - - support_dists = ('SuSE', - 'debian', - 'fedora', - 'redhat', - 'centos', - 'meego', - 'moblin', - 'tizen') - try: - (dist, ver, id) = platform.linux_distribution( \ - supported_dists = support_dists) - except: - (dist, ver, id) = platform.dist( \ - supported_dists = support_dists) - - return (dist, ver, id) - -def get_distro_str(): - """Get composited string for current linux distribution - """ - (dist, ver, id) = get_distro() - - if not dist: - return 'Unknown Linux Distro' - else: - distro_str = ' '.join(map(str.strip, (dist, ver, id))) - return distro_str.strip() - -_LOOP_RULE_PTH = None - -def hide_loopdev_presentation(): - udev_rules = "80-prevent-loop-present.rules" - udev_rules_dir = [ - '/usr/lib/udev/rules.d/', - '/lib/udev/rules.d/', - '/etc/udev/rules.d/' - ] - - global _LOOP_RULE_PTH - - for rdir in udev_rules_dir: - if os.path.exists(rdir): - _LOOP_RULE_PTH = os.path.join(rdir, udev_rules) - - if not _LOOP_RULE_PTH: - return - - try: - with open(_LOOP_RULE_PTH, 'w') as wf: - wf.write('KERNEL=="loop*", ENV{UDISKS_PRESENTATION_HIDE}="1"') - - runner.quiet('udevadm trigger') - except: - pass - -def unhide_loopdev_presentation(): - global _LOOP_RULE_PTH - - if not _LOOP_RULE_PTH: - return - - try: - os.unlink(_LOOP_RULE_PTH) - runner.quiet('udevadm trigger') - except: - pass - -def extract_rpm(rpmfile, targetdir): - rpm2cpio = find_binary_path("rpm2cpio") - cpio = find_binary_path("cpio") - - olddir = os.getcwd() - os.chdir(targetdir) - - msger.verbose("Extract rpm file with cpio: %s" % rpmfile) - p1 = subprocess.Popen([rpm2cpio, rpmfile], stdout=subprocess.PIPE) - p2 = subprocess.Popen([cpio, "-idv"], stdin=p1.stdout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (sout, serr) = p2.communicate() - msger.verbose(sout or serr) - - os.chdir(olddir) - -def compressing(fpath, method): - comp_map = { - "gz": "gzip", - "bz2": "bzip2" - } - if method not in comp_map: - raise CreatorError("Unsupport compress format: %s, valid values: %s" - % (method, ','.join(comp_map.keys()))) - cmd = find_binary_path(comp_map[method]) - rc = runner.show([cmd, "-f", fpath]) - if rc: - raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath)) - -def taring(dstfile, target): - import tarfile - basen, ext = os.path.splitext(dstfile) - comp = {".tar": None, - ".gz": "gz", # for .tar.gz - ".bz2": "bz2", # for .tar.bz2 - ".tgz": "gz", - ".tbz": "bz2"}[ext] - - # specify tarball file path - if not comp: - tarpath = dstfile - elif basen.endswith(".tar"): - tarpath = basen - else: - tarpath = basen + ".tar" - wf = tarfile.open(tarpath, 'w') - - if os.path.isdir(target): - for item in os.listdir(target): - wf.add(os.path.join(target, item), item) - else: - wf.add(target, os.path.basename(target)) - wf.close() - - if comp: - compressing(tarpath, comp) - # when dstfile ext is ".tgz" and ".tbz", should rename - if not basen.endswith(".tar"): - shutil.move("%s.%s" % (tarpath, comp), dstfile) - -def ziping(dstfile, target): - import zipfile - wf = zipfile.ZipFile(dstfile, 'w', compression=zipfile.ZIP_DEFLATED) - if os.path.isdir(target): - for item in os.listdir(target): - fpath = os.path.join(target, item) - if not os.path.isfile(fpath): - continue - wf.write(fpath, item, zipfile.ZIP_DEFLATED) - else: - wf.write(target, os.path.basename(target), zipfile.ZIP_DEFLATED) - wf.close() - -pack_formats = { - ".tar": taring, - ".tar.gz": taring, - ".tar.bz2": taring, - ".tgz": taring, - ".tbz": taring, - ".zip": ziping, -} - -def packing(dstfile, target): - (base, ext) = os.path.splitext(dstfile) - if ext in (".gz", ".bz2") and base.endswith(".tar"): - ext = ".tar" + ext - if ext not in pack_formats: - raise CreatorError("Unsupport pack format: %s, valid values: %s" - % (ext, ','.join(pack_formats.keys()))) - func = pack_formats[ext] - # func should be callable - func(dstfile, target) - -def human_size(size): - """Return human readable string for Bytes size - """ - - if size <= 0: - return "0M" - import math - measure = ['B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'] - expo = int(math.log(size, 1024)) - mant = float(size/math.pow(1024, expo)) - return "{0:.1f}{1:s}".format(mant, measure[expo]) - -def get_block_size(file_obj): - """ Returns block size for file object 'file_obj'. Errors are indicated by - the 'IOError' exception. """ - - from fcntl import ioctl - import struct - - # Get the block size of the host file-system for the image file by calling - # the FIGETBSZ ioctl (number 2). - binary_data = ioctl(file_obj, 2, struct.pack('I', 0)) - return struct.unpack('I', binary_data)[0] - -def check_space_pre_cp(src, dst): - """Check whether disk space is enough before 'cp' like - operations, else exception will be raised. - """ - - srcsize = get_file_size(src) * 1024 * 1024 - freesize = get_filesystem_avail(dst) - if srcsize > freesize: - raise CreatorError("space on %s(%s) is not enough for about %s files" - % (dst, human_size(freesize), human_size(srcsize))) - -def calc_hashes(file_path, hash_names, start = 0, end = None): - """ Calculate hashes for a file. The 'file_path' argument is the file - to calculate hash functions for, 'start' and 'end' are the starting and - ending file offset to calculate the has functions for. The 'hash_names' - argument is a list of hash names to calculate. Returns the the list - of calculated hash values in the hexadecimal form in the same order - as 'hash_names'. - """ - if end == None: - end = os.path.getsize(file_path) - - chunk_size = 65536 - to_read = end - start - read = 0 - - hashes = [] - for hash_name in hash_names: - hashes.append(hashlib.new(hash_name)) - - with open(file_path, "rb") as f: - f.seek(start) - - while read < to_read: - if read + chunk_size > to_read: - chunk_size = to_read - read - chunk = f.read(chunk_size) - for hash_obj in hashes: - hash_obj.update(chunk) - read += chunk_size - - result = [] - for hash_obj in hashes: - result.append(hash_obj.hexdigest()) - - return result - -def get_md5sum(fpath): - return calc_hashes(fpath, ('md5', ))[0] - - -def normalize_ksfile(ksconf, release, arch): - ''' - Return the name of a normalized ks file in which macro variables - @BUILD_ID@ and @ARCH@ are replace with real values. - - The original ks file is returned if no special macro is used, otherwise - a temp file is created and returned, which will be deleted when program - exits normally. - ''' - - if not release: - release = "latest" - if not arch or re.match(r'i.86', arch): - arch = "ia32" - - with open(ksconf) as f: - ksc = f.read() - - if "@ARCH@" not in ksc and "@BUILD_ID@" not in ksc: - return ksconf - - msger.info("Substitute macro variable @BUILD_ID@/@ARCH@ in ks: %s" % ksconf) - ksc = ksc.replace("@ARCH@", arch) - ksc = ksc.replace("@BUILD_ID@", release) - - fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf)) - os.write(fd, ksc) - os.close(fd) - - msger.debug('normalized ks file:%s' % ksconf) - - def remove_temp_ks(): - try: - os.unlink(ksconf) - except OSError, err: - msger.warning('Failed to remove temp ks file:%s:%s' % (ksconf, err)) - - import atexit - atexit.register(remove_temp_ks) - - return ksconf - - -def _check_mic_chroot(rootdir): - def _path(path): - return rootdir.rstrip('/') + path - - release_files = map(_path, [ "/etc/moblin-release", - "/etc/meego-release", - "/etc/tizen-release"]) - - if not any(map(os.path.exists, release_files)): - msger.warning("Dir %s is not a MeeGo/Tizen chroot env" % rootdir) - - if not glob.glob(rootdir + "/boot/vmlinuz-*"): - msger.warning("Failed to find kernel module under %s" % rootdir) - - return - -def selinux_check(arch, fstypes): - try: - getenforce = find_binary_path('getenforce') - except CreatorError: - return - - selinux_status = runner.outs([getenforce]) - if arch and arch.startswith("arm") and selinux_status == "Enforcing": - raise CreatorError("Can't create arm image if selinux is enabled, " - "please run 'setenforce 0' to disable selinux") - - use_btrfs = filter(lambda typ: typ == 'btrfs', fstypes) - if use_btrfs and selinux_status == "Enforcing": - raise CreatorError("Can't create btrfs image if selinux is enabled," - " please run 'setenforce 0' to disable selinux") - -def get_image_type(path): - def _get_extension_name(path): - match = re.search("(?<=\.)\w+$", path) - if match: - return match.group(0) - else: - return None - - if os.path.isdir(path): - _check_mic_chroot(path) - return "fs" - - maptab = { - "tar": "loop", - "raw":"raw", - "vmdk":"vmdk", - "vdi":"vdi", - "iso":"livecd", - "usbimg":"liveusb", - } - - extension = _get_extension_name(path) - if extension in maptab: - return maptab[extension] - - fd = open(path, "rb") - file_header = fd.read(1024) - fd.close() - vdi_flag = "<<< Sun VirtualBox Disk Image >>>" - if file_header[0:len(vdi_flag)] == vdi_flag: - return maptab["vdi"] - - output = runner.outs(['file', path]) - isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*") - usbimgptn = re.compile(r".*x86 boot sector.*active.*") - rawptn = re.compile(r".*x86 boot sector.*") - vmdkptn = re.compile(r".*VMware. disk image.*") - ext3fsimgptn = re.compile(r".*Linux.*ext3 filesystem data.*") - ext4fsimgptn = re.compile(r".*Linux.*ext4 filesystem data.*") - btrfsimgptn = re.compile(r".*BTRFS.*") - if isoptn.match(output): - return maptab["iso"] - elif usbimgptn.match(output): - return maptab["usbimg"] - elif rawptn.match(output): - return maptab["raw"] - elif vmdkptn.match(output): - return maptab["vmdk"] - elif ext3fsimgptn.match(output): - return "ext3fsimg" - elif ext4fsimgptn.match(output): - return "ext4fsimg" - elif btrfsimgptn.match(output): - return "btrfsimg" - else: - raise CreatorError("Cannot detect the type of image: %s" % path) - - -def get_file_size(filename): - """ Return size in MB unit """ - cmd = ['du', "-s", "-b", "-B", "1M", filename] - rc, duOutput = runner.runtool(cmd) - if rc != 0: - raise CreatorError("Failed to run: %s" % ' '.join(cmd)) - size1 = int(duOutput.split()[0]) - - cmd = ['du', "-s", "-B", "1M", filename] - rc, duOutput = runner.runtool(cmd) - if rc != 0: - raise CreatorError("Failed to run: %s" % ' '.join(cmd)) - - size2 = int(duOutput.split()[0]) - return max(size1, size2) - - -def get_filesystem_avail(fs): - vfstat = os.statvfs(fs) - return vfstat.f_bavail * vfstat.f_bsize - -def convert_image(srcimg, srcfmt, dstimg, dstfmt): - #convert disk format - if dstfmt != "raw": - raise CreatorError("Invalid destination image format: %s" % dstfmt) - msger.debug("converting %s image to %s" % (srcimg, dstimg)) - if srcfmt == "vmdk": - path = find_binary_path("qemu-img") - argv = [path, "convert", "-f", "vmdk", srcimg, "-O", dstfmt, dstimg] - elif srcfmt == "vdi": - path = find_binary_path("VBoxManage") - argv = [path, "internalcommands", "converttoraw", srcimg, dstimg] - else: - raise CreatorError("Invalid soure image format: %s" % srcfmt) - - rc = runner.show(argv) - if rc == 0: - msger.debug("convert successful") - if rc != 0: - raise CreatorError("Unable to convert disk to %s" % dstfmt) - -def uncompress_squashfs(squashfsimg, outdir): - """Uncompress file system from squshfs image""" - unsquashfs = find_binary_path("unsquashfs") - args = [ unsquashfs, "-d", outdir, squashfsimg ] - rc = runner.show(args) - if (rc != 0): - raise SquashfsError("Failed to uncompress %s." % squashfsimg) - -def mkdtemp(dir = "/var/tmp", prefix = "wic-tmp-"): - """ FIXME: use the dir in wic.conf instead """ - - makedirs(dir) - return tempfile.mkdtemp(dir = dir, prefix = prefix) - -def get_repostrs_from_ks(ks): - def _get_temp_reponame(baseurl): - md5obj = hashlib.md5(baseurl) - tmpreponame = "%s" % md5obj.hexdigest() - return tmpreponame - - kickstart_repos = [] - - for repodata in ks.handler.repo.repoList: - repo = {} - for attr in ('name', - 'baseurl', - 'mirrorlist', - 'includepkgs', # val is list - 'excludepkgs', # val is list - 'cost', # int - 'priority',# int - 'save', - 'proxy', - 'proxyuser', - 'proxypasswd', - 'proxypasswd', - 'debuginfo', - 'source', - 'gpgkey', - 'ssl_verify'): - if hasattr(repodata, attr) and getattr(repodata, attr): - repo[attr] = getattr(repodata, attr) - - if 'name' not in repo: - repo['name'] = _get_temp_reponame(repodata.baseurl) - - kickstart_repos.append(repo) - - return kickstart_repos - -def _get_uncompressed_data_from_url(url, filename, proxies): - filename = myurlgrab(url, filename, proxies) - suffix = None - if filename.endswith(".gz"): - suffix = ".gz" - runner.quiet(['gunzip', "-f", filename]) - elif filename.endswith(".bz2"): - suffix = ".bz2" - runner.quiet(['bunzip2', "-f", filename]) - if suffix: - filename = filename.replace(suffix, "") - return filename - -def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename, - sumtype=None, checksum=None): - url = os.path.join(baseurl, filename) - filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename))) - if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"): - filename = os.path.splitext(filename_tmp)[0] - else: - filename = filename_tmp - if sumtype and checksum and os.path.exists(filename): - try: - sumcmd = find_binary_path("%ssum" % sumtype) - except: - file_checksum = None - else: - file_checksum = runner.outs([sumcmd, filename]).split()[0] - - if file_checksum and file_checksum == checksum: - return filename - - return _get_uncompressed_data_from_url(url,filename_tmp,proxies) - -def get_metadata_from_repos(repos, cachedir): - my_repo_metadata = [] - for repo in repos: - reponame = repo['name'] - baseurl = repo['baseurl'] - - - if 'proxy' in repo: - proxy = repo['proxy'] - else: - proxy = get_proxy_for(baseurl) - - proxies = None - if proxy: - proxies = {str(baseurl.split(":")[0]):str(proxy)} - - makedirs(os.path.join(cachedir, reponame)) - url = os.path.join(baseurl, "repodata/repomd.xml") - filename = os.path.join(cachedir, reponame, 'repomd.xml') - repomd = myurlgrab(url, filename, proxies) - try: - root = xmlparse(repomd) - except SyntaxError: - raise CreatorError("repomd.xml syntax error.") - - ns = root.getroot().tag - ns = ns[0:ns.rindex("}")+1] - - filepaths = {} - checksums = {} - sumtypes = {} - - for elm in root.getiterator("%sdata" % ns): - if elm.attrib["type"] == "patterns": - filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href'] - checksums['patterns'] = elm.find("%sopen-checksum" % ns).text - sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type'] - break - - for elm in root.getiterator("%sdata" % ns): - if elm.attrib["type"] in ("group_gz", "group"): - filepaths['comps'] = elm.find("%slocation" % ns).attrib['href'] - checksums['comps'] = elm.find("%sopen-checksum" % ns).text - sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type'] - break - - primary_type = None - for elm in root.getiterator("%sdata" % ns): - if elm.attrib["type"] in ("primary_db", "primary"): - primary_type = elm.attrib["type"] - filepaths['primary'] = elm.find("%slocation" % ns).attrib['href'] - checksums['primary'] = elm.find("%sopen-checksum" % ns).text - sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type'] - break - - if not primary_type: - continue - - for item in ("primary", "patterns", "comps"): - if item not in filepaths: - filepaths[item] = None - continue - if not filepaths[item]: - continue - filepaths[item] = _get_metadata_from_repo(baseurl, - proxies, - cachedir, - reponame, - filepaths[item], - sumtypes[item], - checksums[item]) - - """ Get repo key """ - try: - repokey = _get_metadata_from_repo(baseurl, - proxies, - cachedir, - reponame, - "repodata/repomd.xml.key") - except CreatorError: - repokey = None - msger.debug("\ncan't get %s/%s" % (baseurl, "repodata/repomd.xml.key")) - - my_repo_metadata.append({"name":reponame, - "baseurl":baseurl, - "repomd":repomd, - "primary":filepaths['primary'], - "cachedir":cachedir, - "proxies":proxies, - "patterns":filepaths['patterns'], - "comps":filepaths['comps'], - "repokey":repokey}) - - return my_repo_metadata - -def get_rpmver_in_repo(repometadata): - for repo in repometadata: - if repo["primary"].endswith(".xml"): - root = xmlparse(repo["primary"]) - ns = root.getroot().tag - ns = ns[0:ns.rindex("}")+1] - - versionlist = [] - for elm in root.getiterator("%spackage" % ns): - if elm.find("%sname" % ns).text == 'rpm': - for node in elm.getchildren(): - if node.tag == "%sversion" % ns: - versionlist.append(node.attrib['ver']) - - if versionlist: - return reversed( - sorted( - versionlist, - key = lambda ver: map(int, ver.split('.')))).next() - - elif repo["primary"].endswith(".sqlite"): - con = sqlite.connect(repo["primary"]) - for row in con.execute("select version from packages where " - "name=\"rpm\" ORDER by version DESC"): - con.close() - return row[0] - - return None - -def get_arch(repometadata): - archlist = [] - for repo in repometadata: - if repo["primary"].endswith(".xml"): - root = xmlparse(repo["primary"]) - ns = root.getroot().tag - ns = ns[0:ns.rindex("}")+1] - for elm in root.getiterator("%spackage" % ns): - if elm.find("%sarch" % ns).text not in ("noarch", "src"): - arch = elm.find("%sarch" % ns).text - if arch not in archlist: - archlist.append(arch) - elif repo["primary"].endswith(".sqlite"): - con = sqlite.connect(repo["primary"]) - for row in con.execute("select arch from packages where arch not in (\"src\", \"noarch\")"): - if row[0] not in archlist: - archlist.append(row[0]) - - con.close() - - uniq_arch = [] - for i in range(len(archlist)): - if archlist[i] not in rpmmisc.archPolicies.keys(): - continue - need_append = True - j = 0 - while j < len(uniq_arch): - if archlist[i] in rpmmisc.archPolicies[uniq_arch[j]].split(':'): - need_append = False - break - if uniq_arch[j] in rpmmisc.archPolicies[archlist[i]].split(':'): - if need_append: - uniq_arch[j] = archlist[i] - need_append = False - else: - uniq_arch.remove(uniq_arch[j]) - continue - j += 1 - if need_append: - uniq_arch.append(archlist[i]) - - return uniq_arch, archlist - -def get_package(pkg, repometadata, arch = None): - ver = "" - target_repo = None - if not arch: - arches = [] - elif arch not in rpmmisc.archPolicies: - arches = [arch] - else: - arches = rpmmisc.archPolicies[arch].split(':') - arches.append('noarch') - - for repo in repometadata: - if repo["primary"].endswith(".xml"): - root = xmlparse(repo["primary"]) - ns = root.getroot().tag - ns = ns[0:ns.rindex("}")+1] - for elm in root.getiterator("%spackage" % ns): - if elm.find("%sname" % ns).text == pkg: - if elm.find("%sarch" % ns).text in arches: - version = elm.find("%sversion" % ns) - tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel']) - if tmpver > ver: - ver = tmpver - location = elm.find("%slocation" % ns) - pkgpath = "%s" % location.attrib['href'] - target_repo = repo - break - if repo["primary"].endswith(".sqlite"): - con = sqlite.connect(repo["primary"]) - if arch: - sql = 'select version, release, location_href from packages ' \ - 'where name = "%s" and arch IN ("%s")' % \ - (pkg, '","'.join(arches)) - for row in con.execute(sql): - tmpver = "%s-%s" % (row[0], row[1]) - if tmpver > ver: - ver = tmpver - pkgpath = "%s" % row[2] - target_repo = repo - break - else: - sql = 'select version, release, location_href from packages ' \ - 'where name = "%s"' % pkg - for row in con.execute(sql): - tmpver = "%s-%s" % (row[0], row[1]) - if tmpver > ver: - ver = tmpver - pkgpath = "%s" % row[2] - target_repo = repo - break - con.close() - if target_repo: - makedirs("%s/packages/%s" % (target_repo["cachedir"], target_repo["name"])) - url = os.path.join(target_repo["baseurl"], pkgpath) - filename = str("%s/packages/%s/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath))) - if os.path.exists(filename): - ret = rpmmisc.checkRpmIntegrity('rpm', filename) - if ret == 0: - return filename - - msger.warning("package %s is damaged: %s" % - (os.path.basename(filename), filename)) - os.unlink(filename) - - pkg = myurlgrab(str(url), filename, target_repo["proxies"]) - return pkg - else: - return None - -def get_source_name(pkg, repometadata): - - def get_bin_name(pkg): - m = RPM_RE.match(pkg) - if m: - return m.group(1) - return None - - def get_src_name(srpm): - m = SRPM_RE.match(srpm) - if m: - return m.group(1) - return None - - ver = "" - target_repo = None - - pkg_name = get_bin_name(pkg) - if not pkg_name: - return None - - for repo in repometadata: - if repo["primary"].endswith(".xml"): - root = xmlparse(repo["primary"]) - ns = root.getroot().tag - ns = ns[0:ns.rindex("}")+1] - for elm in root.getiterator("%spackage" % ns): - if elm.find("%sname" % ns).text == pkg_name: - if elm.find("%sarch" % ns).text != "src": - version = elm.find("%sversion" % ns) - tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel']) - if tmpver > ver: - ver = tmpver - fmt = elm.find("%sformat" % ns) - if fmt: - fns = fmt.getchildren()[0].tag - fns = fns[0:fns.rindex("}")+1] - pkgpath = fmt.find("%ssourcerpm" % fns).text - target_repo = repo - break - - if repo["primary"].endswith(".sqlite"): - con = sqlite.connect(repo["primary"]) - for row in con.execute("select version, release, rpm_sourcerpm from packages where name = \"%s\" and arch != \"src\"" % pkg_name): - tmpver = "%s-%s" % (row[0], row[1]) - if tmpver > ver: - pkgpath = "%s" % row[2] - target_repo = repo - break - con.close() - if target_repo: - return get_src_name(pkgpath) - else: - return None - -def get_pkglist_in_patterns(group, patterns): - found = False - pkglist = [] - try: - root = xmlparse(patterns) - except SyntaxError: - raise SyntaxError("%s syntax error." % patterns) - - for elm in list(root.getroot()): - ns = elm.tag - ns = ns[0:ns.rindex("}")+1] - name = elm.find("%sname" % ns) - summary = elm.find("%ssummary" % ns) - if name.text == group or summary.text == group: - found = True - break - - if not found: - return pkglist - - found = False - for requires in list(elm): - if requires.tag.endswith("requires"): - found = True - break - - if not found: - return pkglist - - for pkg in list(requires): - pkgname = pkg.attrib["name"] - if pkgname not in pkglist: - pkglist.append(pkgname) - - return pkglist - -def get_pkglist_in_comps(group, comps): - found = False - pkglist = [] - try: - root = xmlparse(comps) - except SyntaxError: - raise SyntaxError("%s syntax error." % comps) - - for elm in root.getiterator("group"): - id = elm.find("id") - name = elm.find("name") - if id.text == group or name.text == group: - packagelist = elm.find("packagelist") - found = True - break - - if not found: - return pkglist - - for require in elm.getiterator("packagereq"): - if require.tag.endswith("packagereq"): - pkgname = require.text - if pkgname not in pkglist: - pkglist.append(pkgname) - - return pkglist - -def is_statically_linked(binary): - return ", statically linked, " in runner.outs(['file', binary]) - -def setup_qemu_emulator(rootdir, arch): - # mount binfmt_misc if it doesn't exist - if not os.path.exists("/proc/sys/fs/binfmt_misc"): - modprobecmd = find_binary_path("modprobe") - runner.show([modprobecmd, "binfmt_misc"]) - if not os.path.exists("/proc/sys/fs/binfmt_misc/register"): - mountcmd = find_binary_path("mount") - runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"]) - - # qemu_emulator is a special case, we can't use find_binary_path - # qemu emulator should be a statically-linked executable file - qemu_emulator = "/usr/bin/qemu-arm" - if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator): - qemu_emulator = "/usr/bin/qemu-arm-static" - if not os.path.exists(qemu_emulator): - raise CreatorError("Please install a statically-linked qemu-arm") - - # qemu emulator version check - armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')] - if arch in armv7_list: # need qemu (>=0.13.0) - qemuout = runner.outs([qemu_emulator, "-h"]) - m = re.search("version\s*([.\d]+)", qemuout) - if m: - qemu_version = m.group(1) - if qemu_version < "0.13": - raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch)) - else: - msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator) - - if not os.path.exists(rootdir + "/usr/bin"): - makedirs(rootdir + "/usr/bin") - shutil.copy(qemu_emulator, rootdir + "/usr/bin/qemu-arm-static") - qemu_emulator = "/usr/bin/qemu-arm-static" - - # disable selinux, selinux will block qemu emulator to run - if os.path.exists("/usr/sbin/setenforce"): - msger.info('Try to disable selinux') - runner.show(["/usr/sbin/setenforce", "0"]) - - # unregister it if it has been registered and is a dynamically-linked executable - node = "/proc/sys/fs/binfmt_misc/arm" - if os.path.exists(node): - qemu_unregister_string = "-1\n" - fd = open("/proc/sys/fs/binfmt_misc/arm", "w") - fd.write(qemu_unregister_string) - fd.close() - - # register qemu emulator for interpreting other arch executable file - if not os.path.exists(node): - qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator - fd = open("/proc/sys/fs/binfmt_misc/register", "w") - fd.write(qemu_arm_string) - fd.close() - - return qemu_emulator - -def SrcpkgsDownload(pkgs, repometadata, instroot, cachedir): - def get_source_repometadata(repometadata): - src_repometadata=[] - for repo in repometadata: - if repo["name"].endswith("-source"): - src_repometadata.append(repo) - if src_repometadata: - return src_repometadata - return None - - def get_src_name(srpm): - m = SRPM_RE.match(srpm) - if m: - return m.group(1) - return None - - src_repometadata = get_source_repometadata(repometadata) - - if not src_repometadata: - msger.warning("No source repo found") - return None - - src_pkgs = [] - lpkgs_dict = {} - lpkgs_path = [] - for repo in src_repometadata: - cachepath = "%s/%s/packages/*.src.rpm" %(cachedir, repo["name"]) - lpkgs_path += glob.glob(cachepath) - - for lpkg in lpkgs_path: - lpkg_name = get_src_name(os.path.basename(lpkg)) - lpkgs_dict[lpkg_name] = lpkg - localpkgs = lpkgs_dict.keys() - - cached_count = 0 - destdir = instroot+'/usr/src/SRPMS' - if not os.path.exists(destdir): - os.makedirs(destdir) - - srcpkgset = set() - for _pkg in pkgs: - srcpkg_name = get_source_name(_pkg, repometadata) - if not srcpkg_name: - continue - srcpkgset.add(srcpkg_name) - - for pkg in list(srcpkgset): - if pkg in localpkgs: - cached_count += 1 - shutil.copy(lpkgs_dict[pkg], destdir) - src_pkgs.append(os.path.basename(lpkgs_dict[pkg])) - else: - src_pkg = get_package(pkg, src_repometadata, 'src') - if src_pkg: - shutil.copy(src_pkg, destdir) - src_pkgs.append(src_pkg) - msger.info("%d source packages gotten from cache" % cached_count) - - return src_pkgs - -def strip_end(text, suffix): - if not text.endswith(suffix): - return text - return text[:-len(suffix)] diff --git a/scripts/lib/mic/utils/oe/__init__.py b/scripts/lib/mic/utils/oe/__init__.py deleted file mode 100644 index d10e802116..0000000000 --- a/scripts/lib/mic/utils/oe/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# OpenEmbedded mic utils library -# -# Copyright (c) 2013, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# AUTHORS -# Tom Zanussi <tom.zanussi (at] linux.intel.com> -# diff --git a/scripts/lib/mic/utils/oe/misc.py b/scripts/lib/mic/utils/oe/misc.py deleted file mode 100644 index 16c250aa9f..0000000000 --- a/scripts/lib/mic/utils/oe/misc.py +++ /dev/null @@ -1,174 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# Copyright (c) 2013, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# DESCRIPTION -# This module provides a place to collect various mic-related utils -# for the OpenEmbedded Image Tools. -# -# AUTHORS -# Tom Zanussi <tom.zanussi (at] linux.intel.com> -# - -from mic import msger -from mic.utils import runner - -def exec_cmd(cmd_and_args, as_shell = False, catch = 3): - """ - Execute command, catching stderr, stdout - - Need to execute as_shell if the command uses wildcards - """ - msger.debug("exec_cmd: %s" % cmd_and_args) - args = cmd_and_args.split() - msger.debug(args) - - if (as_shell): - rc, out = runner.runtool(cmd_and_args, catch) - else: - rc, out = runner.runtool(args, catch) - out = out.strip() - msger.debug("exec_cmd: output for %s (rc = %d): %s" % \ - (cmd_and_args, rc, out)) - - if rc != 0: - # We don't throw exception when return code is not 0, because - # parted always fails to reload part table with loop devices. This - # prevents us from distinguishing real errors based on return - # code. - msger.warning("WARNING: %s returned '%s' instead of 0" % (cmd_and_args, rc)) - - return (rc, out) - - -def exec_cmd_quiet(cmd_and_args, as_shell = False): - """ - Execute command, catching nothing in the output - - Need to execute as_shell if the command uses wildcards - """ - return exec_cmd(cmd_and_args, as_shell, 0) - - -def exec_native_cmd(cmd_and_args, native_sysroot, catch = 3): - """ - Execute native command, catching stderr, stdout - - Need to execute as_shell if the command uses wildcards - - Always need to execute native commands as_shell - """ - native_paths = \ - "export PATH=%s/sbin:%s/usr/sbin:%s/usr/bin:$PATH" % \ - (native_sysroot, native_sysroot, native_sysroot) - native_cmd_and_args = "%s;%s" % (native_paths, cmd_and_args) - msger.debug("exec_native_cmd: %s" % cmd_and_args) - - args = cmd_and_args.split() - msger.debug(args) - - rc, out = exec_cmd(native_cmd_and_args, True, catch) - - if rc == 127: # shell command-not-found - msger.error("A native (host) program required to build the image " - "was not found (see details above). Please make sure " - "it's installed and try again.") - - return (rc, out) - - -def exec_native_cmd_quiet(cmd_and_args, native_sysroot): - """ - Execute native command, catching nothing in the output - - Need to execute as_shell if the command uses wildcards - - Always need to execute native commands as_shell - """ - return exec_native_cmd(cmd_and_args, native_sysroot, 0) - - -# kickstart doesn't support variable substution in commands, so this -# is our current simplistic scheme for supporting that - -wks_vars = dict() - -def get_wks_var(key): - return wks_vars[key] - -def add_wks_var(key, val): - wks_vars[key] = val - -BOOTDD_EXTRA_SPACE = 16384 -IMAGE_EXTRA_SPACE = 10240 - -__bitbake_env_lines = "" - -def set_bitbake_env_lines(bitbake_env_lines): - global __bitbake_env_lines - __bitbake_env_lines = bitbake_env_lines - -def get_bitbake_env_lines(): - return __bitbake_env_lines - -def find_bitbake_env_lines(image_name): - """ - If image_name is empty, plugins might still be able to use the - environment, so set it regardless. - """ - if image_name: - bitbake_env_cmd = "bitbake -e %s" % image_name - else: - bitbake_env_cmd = "bitbake -e" - rc, bitbake_env_lines = exec_cmd(bitbake_env_cmd) - if rc != 0: - print "Couldn't get '%s' output." % bitbake_env_cmd - return None - - return bitbake_env_lines - -def find_artifact(bitbake_env_lines, variable): - """ - Gather the build artifact for the current image (the image_name - e.g. core-image-minimal) for the current MACHINE set in local.conf - """ - retval = "" - - for line in bitbake_env_lines.split('\n'): - if (get_line_val(line, variable)): - retval = get_line_val(line, variable) - break - - return retval - -def get_line_val(line, key): - """ - Extract the value from the VAR="val" string - """ - if line.startswith(key + "="): - stripped_line = line.split('=')[1] - stripped_line = stripped_line.replace('\"', '') - return stripped_line - return None - -def get_bitbake_var(key): - for line in __bitbake_env_lines.split('\n'): - if (get_line_val(line, key)): - val = get_line_val(line, key) - return val - return None diff --git a/scripts/lib/mic/utils/partitionedfs.py b/scripts/lib/mic/utils/partitionedfs.py deleted file mode 100644 index 593cf1f317..0000000000 --- a/scripts/lib/mic/utils/partitionedfs.py +++ /dev/null @@ -1,782 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2009, 2010, 2011 Intel, Inc. -# Copyright (c) 2007, 2008 Red Hat, Inc. -# Copyright (c) 2008 Daniel P. Berrange -# Copyright (c) 2008 David P. Huff -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os - -from mic import msger -from mic.utils import runner -from mic.utils.errors import MountError -from mic.utils.fs_related import * -from mic.utils.gpt_parser import GptParser -from mic.utils.oe.misc import * - -# Overhead of the MBR partitioning scheme (just one sector) -MBR_OVERHEAD = 1 -# Overhead of the GPT partitioning scheme -GPT_OVERHEAD = 34 - -# Size of a sector in bytes -SECTOR_SIZE = 512 - -class PartitionedMount(Mount): - def __init__(self, mountdir, skipformat = False): - Mount.__init__(self, mountdir) - self.disks = {} - self.partitions = [] - self.subvolumes = [] - self.mapped = False - self.mountOrder = [] - self.unmountOrder = [] - self.parted = find_binary_path("parted") - self.btrfscmd=None - self.skipformat = skipformat - self.snapshot_created = self.skipformat - # Size of a sector used in calculations - self.sector_size = SECTOR_SIZE - self._partitions_layed_out = False - - def __add_disk(self, disk_name): - """ Add a disk 'disk_name' to the internal list of disks. Note, - 'disk_name' is the name of the disk in the target system - (e.g., sdb). """ - - if disk_name in self.disks: - # We already have this disk - return - - assert not self._partitions_layed_out - - self.disks[disk_name] = \ - { 'disk': None, # Disk object - 'mapped': False, # True if kpartx mapping exists - 'numpart': 0, # Number of allocate partitions - 'partitions': [], # Indexes to self.partitions - 'offset': 0, # Offset of next partition (in sectors) - # Minimum required disk size to fit all partitions (in bytes) - 'min_size': 0, - 'ptable_format': "msdos" } # Partition table format - - def add_disk(self, disk_name, disk_obj): - """ Add a disk object which have to be partitioned. More than one disk - can be added. In case of multiple disks, disk partitions have to be - added for each disk separately with 'add_partition()". """ - - self.__add_disk(disk_name) - self.disks[disk_name]['disk'] = disk_obj - - def __add_partition(self, part): - """ This is a helper function for 'add_partition()' which adds a - partition to the internal list of partitions. """ - - assert not self._partitions_layed_out - - self.partitions.append(part) - self.__add_disk(part['disk_name']) - - def add_partition(self, size, disk_name, mountpoint, source_file = None, fstype = None, - label=None, fsopts = None, boot = False, align = None, - part_type = None): - """ Add the next partition. Prtitions have to be added in the - first-to-last order. """ - - ks_pnum = len(self.partitions) - - # Converting MB to sectors for parted - size = size * 1024 * 1024 / self.sector_size - - # We need to handle subvolumes for btrfs - if fstype == "btrfs" and fsopts and fsopts.find("subvol=") != -1: - self.btrfscmd=find_binary_path("btrfs") - subvol = None - opts = fsopts.split(",") - for opt in opts: - if opt.find("subvol=") != -1: - subvol = opt.replace("subvol=", "").strip() - break - if not subvol: - raise MountError("No subvolume: %s" % fsopts) - self.subvolumes.append({'size': size, # In sectors - 'mountpoint': mountpoint, # Mount relative to chroot - 'fstype': fstype, # Filesystem type - 'fsopts': fsopts, # Filesystem mount options - 'disk_name': disk_name, # physical disk name holding partition - 'device': None, # kpartx device node for partition - 'mount': None, # Mount object - 'subvol': subvol, # Subvolume name - 'boot': boot, # Bootable flag - 'mounted': False # Mount flag - }) - - # We still need partition for "/" or non-subvolume - if mountpoint == "/" or not fsopts or fsopts.find("subvol=") == -1: - # Don't need subvolume for "/" because it will be set as default subvolume - if fsopts and fsopts.find("subvol=") != -1: - opts = fsopts.split(",") - for opt in opts: - if opt.strip().startswith("subvol="): - opts.remove(opt) - break - fsopts = ",".join(opts) - - part = { 'ks_pnum' : ks_pnum, # Partition number in the KS file - 'size': size, # In sectors - 'mountpoint': mountpoint, # Mount relative to chroot - 'source_file': source_file, # partition contents - 'fstype': fstype, # Filesystem type - 'fsopts': fsopts, # Filesystem mount options - 'label': label, # Partition label - 'disk_name': disk_name, # physical disk name holding partition - 'device': None, # kpartx device node for partition - 'mount': None, # Mount object - 'num': None, # Partition number - 'boot': boot, # Bootable flag - 'align': align, # Partition alignment - 'part_type' : part_type, # Partition type - 'partuuid': None } # Partition UUID (GPT-only) - - self.__add_partition(part) - - def layout_partitions(self, ptable_format = "msdos"): - """ Layout the partitions, meaning calculate the position of every - partition on the disk. The 'ptable_format' parameter defines the - partition table format, and may be either "msdos" or "gpt". """ - - msger.debug("Assigning %s partitions to disks" % ptable_format) - - if ptable_format not in ('msdos', 'gpt'): - raise MountError("Unknown partition table format '%s', supported " \ - "formats are: 'msdos' and 'gpt'" % ptable_format) - - if self._partitions_layed_out: - return - - self._partitions_layed_out = True - - # Go through partitions in the order they are added in .ks file - for n in range(len(self.partitions)): - p = self.partitions[n] - - if not self.disks.has_key(p['disk_name']): - raise MountError("No disk %s for partition %s" \ - % (p['disk_name'], p['mountpoint'])) - - if p['part_type'] and ptable_format != 'gpt': - # The --part-type can also be implemented for MBR partitions, - # in which case it would map to the 1-byte "partition type" - # filed at offset 3 of the partition entry. - raise MountError("setting custom partition type is only " \ - "imlemented for GPT partitions") - - # Get the disk where the partition is located - d = self.disks[p['disk_name']] - d['numpart'] += 1 - d['ptable_format'] = ptable_format - - if d['numpart'] == 1: - if ptable_format == "msdos": - overhead = MBR_OVERHEAD - else: - overhead = GPT_OVERHEAD - - # Skip one sector required for the partitioning scheme overhead - d['offset'] += overhead - # Steal few sectors from the first partition to offset for the - # partitioning overhead - p['size'] -= overhead - - if p['align']: - # If not first partition and we do have alignment set we need - # to align the partition. - # FIXME: This leaves a empty spaces to the disk. To fill the - # gaps we could enlargea the previous partition? - - # Calc how much the alignment is off. - align_sectors = d['offset'] % (p['align'] * 1024 / self.sector_size) - # We need to move forward to the next alignment point - align_sectors = (p['align'] * 1024 / self.sector_size) - align_sectors - - msger.debug("Realignment for %s%s with %s sectors, original" - " offset %s, target alignment is %sK." % - (p['disk_name'], d['numpart'], align_sectors, - d['offset'], p['align'])) - - # increase the offset so we actually start the partition on right alignment - d['offset'] += align_sectors - - p['start'] = d['offset'] - d['offset'] += p['size'] - - p['type'] = 'primary' - p['num'] = d['numpart'] - - if d['ptable_format'] == "msdos": - if d['numpart'] > 2: - # Every logical partition requires an additional sector for - # the EBR, so steal the last sector from the end of each - # partition starting from the 3rd one for the EBR. This - # will make sure the logical partitions are aligned - # correctly. - p['size'] -= 1 - - if d['numpart'] > 3: - p['type'] = 'logical' - p['num'] = d['numpart'] + 1 - - d['partitions'].append(n) - msger.debug("Assigned %s to %s%d, sectors range %d-%d size %d " - "sectors (%d bytes)." \ - % (p['mountpoint'], p['disk_name'], p['num'], - p['start'], p['start'] + p['size'] - 1, - p['size'], p['size'] * self.sector_size)) - - # Once all the partitions have been layed out, we can calculate the - # minumim disk sizes. - for disk_name, d in self.disks.items(): - d['min_size'] = d['offset'] - if d['ptable_format'] == 'gpt': - # Account for the backup partition table at the end of the disk - d['min_size'] += GPT_OVERHEAD - - d['min_size'] *= self.sector_size - - def __run_parted(self, args): - """ Run parted with arguments specified in the 'args' list. """ - - args.insert(0, self.parted) - msger.debug(args) - - rc, out = runner.runtool(args, catch = 3) - out = out.strip() - if out: - msger.debug('"parted" output: %s' % out) - - if rc != 0: - # We don't throw exception when return code is not 0, because - # parted always fails to reload part table with loop devices. This - # prevents us from distinguishing real errors based on return - # code. - msger.error("WARNING: parted returned '%s' instead of 0 (use --debug for details)" % rc) - - def __create_partition(self, device, parttype, fstype, start, size): - """ Create a partition on an image described by the 'device' object. """ - - # Start is included to the size so we need to substract one from the end. - end = start + size - 1 - msger.debug("Added '%s' partition, sectors %d-%d, size %d sectors" % - (parttype, start, end, size)) - - args = ["-s", device, "unit", "s", "mkpart", parttype] - if fstype: - args.extend([fstype]) - args.extend(["%d" % start, "%d" % end]) - - return self.__run_parted(args) - - def __format_disks(self): - self.layout_partitions() - - if self.skipformat: - msger.debug("Skipping disk format, because skipformat flag is set.") - return - - for dev in self.disks.keys(): - d = self.disks[dev] - msger.debug("Initializing partition table for %s" % \ - (d['disk'].device)) - self.__run_parted(["-s", d['disk'].device, "mklabel", - d['ptable_format']]) - - msger.debug("Creating partitions") - - for p in self.partitions: - d = self.disks[p['disk_name']] - if d['ptable_format'] == "msdos" and p['num'] == 5: - # The last sector of the 3rd partition was reserved for the EBR - # of the first _logical_ partition. This is why the extended - # partition should start one sector before the first logical - # partition. - self.__create_partition(d['disk'].device, "extended", - None, p['start'] - 1, - d['offset'] - p['start']) - - if p['fstype'] == "swap": - parted_fs_type = "linux-swap" - elif p['fstype'] == "vfat": - parted_fs_type = "fat32" - elif p['fstype'] == "msdos": - parted_fs_type = "fat16" - else: - # Type for ext2/ext3/ext4/btrfs - parted_fs_type = "ext2" - - # Boot ROM of OMAP boards require vfat boot partition to have an - # even number of sectors. - if p['mountpoint'] == "/boot" and p['fstype'] in ["vfat", "msdos"] \ - and p['size'] % 2: - msger.debug("Substracting one sector from '%s' partition to " \ - "get even number of sectors for the partition" % \ - p['mountpoint']) - p['size'] -= 1 - - self.__create_partition(d['disk'].device, p['type'], - parted_fs_type, p['start'], p['size']) - - if p['boot']: - if d['ptable_format'] == 'gpt': - flag_name = "legacy_boot" - else: - flag_name = "boot" - msger.debug("Set '%s' flag for partition '%s' on disk '%s'" % \ - (flag_name, p['num'], d['disk'].device)) - self.__run_parted(["-s", d['disk'].device, "set", - "%d" % p['num'], flag_name, "on"]) - - # Parted defaults to enabling the lba flag for fat16 partitions, - # which causes compatibility issues with some firmware (and really - # isn't necessary). - if parted_fs_type == "fat16": - if d['ptable_format'] == 'msdos': - msger.debug("Disable 'lba' flag for partition '%s' on disk '%s'" % \ - (p['num'], d['disk'].device)) - self.__run_parted(["-s", d['disk'].device, "set", - "%d" % p['num'], "lba", "off"]) - - # If the partition table format is "gpt", find out PARTUUIDs for all - # the partitions. And if users specified custom parition type UUIDs, - # set them. - for disk_name, disk in self.disks.items(): - if disk['ptable_format'] != 'gpt': - continue - - pnum = 0 - gpt_parser = GptParser(d['disk'].device, SECTOR_SIZE) - # Iterate over all GPT partitions on this disk - for entry in gpt_parser.get_partitions(): - pnum += 1 - # Find the matching partition in the 'self.partitions' list - for n in d['partitions']: - p = self.partitions[n] - if p['num'] == pnum: - # Found, fetch PARTUUID (partition's unique ID) - p['partuuid'] = entry['part_uuid'] - msger.debug("PARTUUID for partition %d on disk '%s' " \ - "(mount point '%s') is '%s'" % (pnum, \ - disk_name, p['mountpoint'], p['partuuid'])) - if p['part_type']: - entry['type_uuid'] = p['part_type'] - msger.debug("Change type of partition %d on disk " \ - "'%s' (mount point '%s') to '%s'" % \ - (pnum, disk_name, p['mountpoint'], - p['part_type'])) - gpt_parser.change_partition(entry) - - del gpt_parser - - def __map_partitions(self): - """Load it if dm_snapshot isn't loaded. """ - load_module("dm_snapshot") - - for dev in self.disks.keys(): - d = self.disks[dev] - if d['mapped']: - continue - - msger.debug("Running kpartx on %s" % d['disk'].device ) - rc, kpartxOutput = runner.runtool([self.kpartx, "-l", "-v", d['disk'].device]) - kpartxOutput = kpartxOutput.splitlines() - - if rc != 0: - raise MountError("Failed to query partition mapping for '%s'" % - d['disk'].device) - - # Strip trailing blank and mask verbose output - i = 0 - while i < len(kpartxOutput) and kpartxOutput[i][0:4] != "loop": - i = i + 1 - kpartxOutput = kpartxOutput[i:] - - # Make sure kpartx reported the right count of partitions - if len(kpartxOutput) != d['numpart']: - # If this disk has more than 3 partitions, then in case of MBR - # paritions there is an extended parition. Different versions - # of kpartx behave differently WRT the extended partition - - # some map it, some ignore it. This is why we do the below hack - # - if kpartx reported one more partition and the partition - # table type is "msdos" and the amount of partitions is more - # than 3, we just assume kpartx mapped the extended parition - # and we remove it. - if len(kpartxOutput) == d['numpart'] + 1 \ - and d['ptable_format'] == 'msdos' and len(kpartxOutput) > 3: - kpartxOutput.pop(3) - else: - raise MountError("Unexpected number of partitions from " \ - "kpartx: %d != %d" % \ - (len(kpartxOutput), d['numpart'])) - - for i in range(len(kpartxOutput)): - line = kpartxOutput[i] - newdev = line.split()[0] - mapperdev = "/dev/mapper/" + newdev - loopdev = d['disk'].device + newdev[-1] - - msger.debug("Dev %s: %s -> %s" % (newdev, loopdev, mapperdev)) - pnum = d['partitions'][i] - self.partitions[pnum]['device'] = loopdev - - # grub's install wants partitions to be named - # to match their parent device + partition num - # kpartx doesn't work like this, so we add compat - # symlinks to point to /dev/mapper - if os.path.lexists(loopdev): - os.unlink(loopdev) - os.symlink(mapperdev, loopdev) - - msger.debug("Adding partx mapping for %s" % d['disk'].device) - rc = runner.show([self.kpartx, "-v", "-a", d['disk'].device]) - - if rc != 0: - # Make sure that the device maps are also removed on error case. - # The d['mapped'] isn't set to True if the kpartx fails so - # failed mapping will not be cleaned on cleanup either. - runner.quiet([self.kpartx, "-d", d['disk'].device]) - raise MountError("Failed to map partitions for '%s'" % - d['disk'].device) - - # FIXME: there is a bit delay for multipath device setup, - # wait 10ms for the setup - import time - time.sleep(10) - d['mapped'] = True - - def __unmap_partitions(self): - for dev in self.disks.keys(): - d = self.disks[dev] - if not d['mapped']: - continue - - msger.debug("Removing compat symlinks") - for pnum in d['partitions']: - if self.partitions[pnum]['device'] != None: - os.unlink(self.partitions[pnum]['device']) - self.partitions[pnum]['device'] = None - - msger.debug("Unmapping %s" % d['disk'].device) - rc = runner.quiet([self.kpartx, "-d", d['disk'].device]) - if rc != 0: - raise MountError("Failed to unmap partitions for '%s'" % - d['disk'].device) - - d['mapped'] = False - - def __calculate_mountorder(self): - msger.debug("Calculating mount order") - for p in self.partitions: - if p['mountpoint']: - self.mountOrder.append(p['mountpoint']) - self.unmountOrder.append(p['mountpoint']) - - self.mountOrder.sort() - self.unmountOrder.sort() - self.unmountOrder.reverse() - - def cleanup(self): - Mount.cleanup(self) - if self.disks: - self.__unmap_partitions() - for dev in self.disks.keys(): - d = self.disks[dev] - try: - d['disk'].cleanup() - except: - pass - - def unmount(self): - self.__unmount_subvolumes() - for mp in self.unmountOrder: - if mp == 'swap': - continue - p = None - for p1 in self.partitions: - if p1['mountpoint'] == mp: - p = p1 - break - - if p['mount'] != None: - try: - # Create subvolume snapshot here - if p['fstype'] == "btrfs" and p['mountpoint'] == "/" and not self.snapshot_created: - self.__create_subvolume_snapshots(p, p["mount"]) - p['mount'].cleanup() - except: - pass - p['mount'] = None - - # Only for btrfs - def __get_subvolume_id(self, rootpath, subvol): - if not self.btrfscmd: - self.btrfscmd=find_binary_path("btrfs") - argv = [ self.btrfscmd, "subvolume", "list", rootpath ] - - rc, out = runner.runtool(argv) - msger.debug(out) - - if rc != 0: - raise MountError("Failed to get subvolume id from %s', return code: %d." % (rootpath, rc)) - - subvolid = -1 - for line in out.splitlines(): - if line.endswith(" path %s" % subvol): - subvolid = line.split()[1] - if not subvolid.isdigit(): - raise MountError("Invalid subvolume id: %s" % subvolid) - subvolid = int(subvolid) - break - return subvolid - - def __create_subvolume_metadata(self, p, pdisk): - if len(self.subvolumes) == 0: - return - - argv = [ self.btrfscmd, "subvolume", "list", pdisk.mountdir ] - rc, out = runner.runtool(argv) - msger.debug(out) - - if rc != 0: - raise MountError("Failed to get subvolume id from %s', return code: %d." % (pdisk.mountdir, rc)) - - subvolid_items = out.splitlines() - subvolume_metadata = "" - for subvol in self.subvolumes: - for line in subvolid_items: - if line.endswith(" path %s" % subvol["subvol"]): - subvolid = line.split()[1] - if not subvolid.isdigit(): - raise MountError("Invalid subvolume id: %s" % subvolid) - - subvolid = int(subvolid) - opts = subvol["fsopts"].split(",") - for opt in opts: - if opt.strip().startswith("subvol="): - opts.remove(opt) - break - fsopts = ",".join(opts) - subvolume_metadata += "%d\t%s\t%s\t%s\n" % (subvolid, subvol["subvol"], subvol['mountpoint'], fsopts) - - if subvolume_metadata: - fd = open("%s/.subvolume_metadata" % pdisk.mountdir, "w") - fd.write(subvolume_metadata) - fd.close() - - def __get_subvolume_metadata(self, p, pdisk): - subvolume_metadata_file = "%s/.subvolume_metadata" % pdisk.mountdir - if not os.path.exists(subvolume_metadata_file): - return - - fd = open(subvolume_metadata_file, "r") - content = fd.read() - fd.close() - - for line in content.splitlines(): - items = line.split("\t") - if items and len(items) == 4: - self.subvolumes.append({'size': 0, # In sectors - 'mountpoint': items[2], # Mount relative to chroot - 'fstype': "btrfs", # Filesystem type - 'fsopts': items[3] + ",subvol=%s" % items[1], # Filesystem mount options - 'disk_name': p['disk_name'], # physical disk name holding partition - 'device': None, # kpartx device node for partition - 'mount': None, # Mount object - 'subvol': items[1], # Subvolume name - 'boot': False, # Bootable flag - 'mounted': False # Mount flag - }) - - def __create_subvolumes(self, p, pdisk): - """ Create all the subvolumes. """ - - for subvol in self.subvolumes: - argv = [ self.btrfscmd, "subvolume", "create", pdisk.mountdir + "/" + subvol["subvol"]] - - rc = runner.show(argv) - if rc != 0: - raise MountError("Failed to create subvolume '%s', return code: %d." % (subvol["subvol"], rc)) - - # Set default subvolume, subvolume for "/" is default - subvol = None - for subvolume in self.subvolumes: - if subvolume["mountpoint"] == "/" and p['disk_name'] == subvolume['disk_name']: - subvol = subvolume - break - - if subvol: - # Get default subvolume id - subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"]) - # Set default subvolume - if subvolid != -1: - rc = runner.show([ self.btrfscmd, "subvolume", "set-default", "%d" % subvolid, pdisk.mountdir]) - if rc != 0: - raise MountError("Failed to set default subvolume id: %d', return code: %d." % (subvolid, rc)) - - self.__create_subvolume_metadata(p, pdisk) - - def __mount_subvolumes(self, p, pdisk): - if self.skipformat: - # Get subvolume info - self.__get_subvolume_metadata(p, pdisk) - # Set default mount options - if len(self.subvolumes) != 0: - for subvol in self.subvolumes: - if subvol["mountpoint"] == p["mountpoint"] == "/": - opts = subvol["fsopts"].split(",") - for opt in opts: - if opt.strip().startswith("subvol="): - opts.remove(opt) - break - pdisk.fsopts = ",".join(opts) - break - - if len(self.subvolumes) == 0: - # Return directly if no subvolumes - return - - # Remount to make default subvolume mounted - rc = runner.show([self.umountcmd, pdisk.mountdir]) - if rc != 0: - raise MountError("Failed to umount %s" % pdisk.mountdir) - - rc = runner.show([self.mountcmd, "-o", pdisk.fsopts, pdisk.disk.device, pdisk.mountdir]) - if rc != 0: - raise MountError("Failed to umount %s" % pdisk.mountdir) - - for subvol in self.subvolumes: - if subvol["mountpoint"] == "/": - continue - subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"]) - if subvolid == -1: - msger.debug("WARNING: invalid subvolume %s" % subvol["subvol"]) - continue - # Replace subvolume name with subvolume ID - opts = subvol["fsopts"].split(",") - for opt in opts: - if opt.strip().startswith("subvol="): - opts.remove(opt) - break - - opts.extend(["subvolrootid=0", "subvol=%s" % subvol["subvol"]]) - fsopts = ",".join(opts) - subvol['fsopts'] = fsopts - mountpoint = self.mountdir + subvol['mountpoint'] - makedirs(mountpoint) - rc = runner.show([self.mountcmd, "-o", fsopts, pdisk.disk.device, mountpoint]) - if rc != 0: - raise MountError("Failed to mount subvolume %s to %s" % (subvol["subvol"], mountpoint)) - subvol["mounted"] = True - - def __unmount_subvolumes(self): - """ It may be called multiple times, so we need to chekc if it is still mounted. """ - for subvol in self.subvolumes: - if subvol["mountpoint"] == "/": - continue - if not subvol["mounted"]: - continue - mountpoint = self.mountdir + subvol['mountpoint'] - rc = runner.show([self.umountcmd, mountpoint]) - if rc != 0: - raise MountError("Failed to unmount subvolume %s from %s" % (subvol["subvol"], mountpoint)) - subvol["mounted"] = False - - def __create_subvolume_snapshots(self, p, pdisk): - import time - - if self.snapshot_created: - return - - # Remount with subvolid=0 - rc = runner.show([self.umountcmd, pdisk.mountdir]) - if rc != 0: - raise MountError("Failed to umount %s" % pdisk.mountdir) - if pdisk.fsopts: - mountopts = pdisk.fsopts + ",subvolid=0" - else: - mountopts = "subvolid=0" - rc = runner.show([self.mountcmd, "-o", mountopts, pdisk.disk.device, pdisk.mountdir]) - if rc != 0: - raise MountError("Failed to umount %s" % pdisk.mountdir) - - # Create all the subvolume snapshots - snapshotts = time.strftime("%Y%m%d-%H%M") - for subvol in self.subvolumes: - subvolpath = pdisk.mountdir + "/" + subvol["subvol"] - snapshotpath = subvolpath + "_%s-1" % snapshotts - rc = runner.show([ self.btrfscmd, "subvolume", "snapshot", subvolpath, snapshotpath ]) - if rc != 0: - raise MountError("Failed to create subvolume snapshot '%s' for '%s', return code: %d." % (snapshotpath, subvolpath, rc)) - - self.snapshot_created = True - - def __install_partition(self, num, source_file, start, size): - """ - Install source_file contents into a partition. - """ - if not source_file: # nothing to install - return - - # Start is included in the size so need to substract one from the end. - end = start + size - 1 - msger.debug("Installed %s in partition %d, sectors %d-%d, size %d sectors" % (source_file, num, start, end, size)) - - dd_cmd = "dd if=%s of=%s bs=%d seek=%d count=%d conv=notrunc" % \ - (source_file, self.image_file, self.sector_size, start, size) - rc, out = exec_cmd(dd_cmd) - - - def install(self, image_file): - msger.debug("Installing partitions") - - self.image_file = image_file - - for p in self.partitions: - d = self.disks[p['disk_name']] - if d['ptable_format'] == "msdos" and p['num'] == 5: - # The last sector of the 3rd partition was reserved for the EBR - # of the first _logical_ partition. This is why the extended - # partition should start one sector before the first logical - # partition. - self.__install_partition(p['num'], p['source_file'], - p['start'] - 1, - d['offset'] - p['start']) - - self.__install_partition(p['num'], p['source_file'], - p['start'], p['size']) - - def mount(self): - for dev in self.disks.keys(): - d = self.disks[dev] - d['disk'].create() - - self.__format_disks() - - self.__calculate_mountorder() - - return - - def resparse(self, size = None): - # Can't re-sparse a disk image - too hard - pass diff --git a/scripts/lib/mic/utils/proxy.py b/scripts/lib/mic/utils/proxy.py deleted file mode 100644 index 91451a2d01..0000000000 --- a/scripts/lib/mic/utils/proxy.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2010, 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import urlparse - -_my_proxies = {} -_my_noproxy = None -_my_noproxy_list = [] - -def set_proxy_environ(): - global _my_noproxy, _my_proxies - if not _my_proxies: - return - for key in _my_proxies.keys(): - os.environ[key + "_proxy"] = _my_proxies[key] - if not _my_noproxy: - return - os.environ["no_proxy"] = _my_noproxy - -def unset_proxy_environ(): - for env in ('http_proxy', - 'https_proxy', - 'ftp_proxy', - 'all_proxy'): - if env in os.environ: - del os.environ[env] - - ENV=env.upper() - if ENV in os.environ: - del os.environ[ENV] - -def _set_proxies(proxy = None, no_proxy = None): - """Return a dictionary of scheme -> proxy server URL mappings. - """ - - global _my_noproxy, _my_proxies - _my_proxies = {} - _my_noproxy = None - proxies = [] - if proxy: - proxies.append(("http_proxy", proxy)) - if no_proxy: - proxies.append(("no_proxy", no_proxy)) - - # Get proxy settings from environment if not provided - if not proxy and not no_proxy: - proxies = os.environ.items() - - # Remove proxy env variables, urllib2 can't handle them correctly - unset_proxy_environ() - - for name, value in proxies: - name = name.lower() - if value and name[-6:] == '_proxy': - if name[0:2] != "no": - _my_proxies[name[:-6]] = value - else: - _my_noproxy = value - -def _ip_to_int(ip): - ipint=0 - shift=24 - for dec in ip.split("."): - ipint |= int(dec) << shift - shift -= 8 - return ipint - -def _int_to_ip(val): - ipaddr="" - shift=0 - for i in range(4): - dec = val >> shift - dec &= 0xff - ipaddr = ".%d%s" % (dec, ipaddr) - shift += 8 - return ipaddr[1:] - -def _isip(host): - if host.replace(".", "").isdigit(): - return True - return False - -def _set_noproxy_list(): - global _my_noproxy, _my_noproxy_list - _my_noproxy_list = [] - if not _my_noproxy: - return - for item in _my_noproxy.split(","): - item = item.strip() - if not item: - continue - - if item[0] != '.' and item.find("/") == -1: - # Need to match it - _my_noproxy_list.append({"match":0,"needle":item}) - - elif item[0] == '.': - # Need to match at tail - _my_noproxy_list.append({"match":1,"needle":item}) - - elif item.find("/") > 3: - # IP/MASK, need to match at head - needle = item[0:item.find("/")].strip() - ip = _ip_to_int(needle) - netmask = 0 - mask = item[item.find("/")+1:].strip() - - if mask.isdigit(): - netmask = int(mask) - netmask = ~((1<<(32-netmask)) - 1) - ip &= netmask - else: - shift=24 - netmask=0 - for dec in mask.split("."): - netmask |= int(dec) << shift - shift -= 8 - ip &= netmask - - _my_noproxy_list.append({"match":2,"needle":ip,"netmask":netmask}) - -def _isnoproxy(url): - (scheme, host, path, parm, query, frag) = urlparse.urlparse(url) - - if '@' in host: - user_pass, host = host.split('@', 1) - - if ':' in host: - host, port = host.split(':', 1) - - hostisip = _isip(host) - for item in _my_noproxy_list: - if hostisip and item["match"] <= 1: - continue - - if item["match"] == 2 and hostisip: - if (_ip_to_int(host) & item["netmask"]) == item["needle"]: - return True - - if item["match"] == 0: - if host == item["needle"]: - return True - - if item["match"] == 1: - if host.rfind(item["needle"]) > 0: - return True - - return False - -def set_proxies(proxy = None, no_proxy = None): - _set_proxies(proxy, no_proxy) - _set_noproxy_list() - set_proxy_environ() - -def get_proxy_for(url): - if url.startswith('file:') or _isnoproxy(url): - return None - - type = url[0:url.index(":")] - proxy = None - if _my_proxies.has_key(type): - proxy = _my_proxies[type] - elif _my_proxies.has_key("http"): - proxy = _my_proxies["http"] - else: - proxy = None - - return proxy diff --git a/scripts/lib/mic/utils/rpmmisc.py b/scripts/lib/mic/utils/rpmmisc.py deleted file mode 100644 index af15763e18..0000000000 --- a/scripts/lib/mic/utils/rpmmisc.py +++ /dev/null @@ -1,600 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2008, 2009, 2010, 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import sys -import re -import rpm - -from mic import msger -from mic.utils.errors import CreatorError -from mic.utils.proxy import get_proxy_for -from mic.utils import runner - - -class RPMInstallCallback: - """ Command line callback class for callbacks from the RPM library. - """ - - def __init__(self, ts, output=1): - self.output = output - self.callbackfilehandles = {} - self.total_actions = 0 - self.total_installed = 0 - self.installed_pkg_names = [] - self.total_removed = 0 - self.mark = "+" - self.marks = 40 - self.lastmsg = None - self.tsInfo = None # this needs to be set for anything else to work - self.ts = ts - self.filelog = False - self.logString = [] - self.headmsg = "Installing" - - def _dopkgtup(self, hdr): - tmpepoch = hdr['epoch'] - if tmpepoch is None: epoch = '0' - else: epoch = str(tmpepoch) - - return (hdr['name'], hdr['arch'], epoch, hdr['version'], hdr['release']) - - def _makeHandle(self, hdr): - handle = '%s:%s.%s-%s-%s' % (hdr['epoch'], hdr['name'], hdr['version'], - hdr['release'], hdr['arch']) - - return handle - - def _localprint(self, msg): - if self.output: - msger.info(msg) - - def _makefmt(self, percent, progress = True): - l = len(str(self.total_actions)) - size = "%s.%s" % (l, l) - fmt_done = "[%" + size + "s/%" + size + "s]" - done = fmt_done % (self.total_installed + self.total_removed, - self.total_actions) - marks = self.marks - (2 * l) - width = "%s.%s" % (marks, marks) - fmt_bar = "%-" + width + "s" - if progress: - bar = fmt_bar % (self.mark * int(marks * (percent / 100.0)), ) - fmt = "\r %-10.10s: %-20.20s " + bar + " " + done - else: - bar = fmt_bar % (self.mark * marks, ) - fmt = " %-10.10s: %-20.20s " + bar + " " + done - return fmt - - def _logPkgString(self, hdr): - """return nice representation of the package for the log""" - (n,a,e,v,r) = self._dopkgtup(hdr) - if e == '0': - pkg = '%s.%s %s-%s' % (n, a, v, r) - else: - pkg = '%s.%s %s:%s-%s' % (n, a, e, v, r) - - return pkg - - def callback(self, what, bytes, total, h, user): - if what == rpm.RPMCALLBACK_TRANS_START: - if bytes == 6: - self.total_actions = total - - elif what == rpm.RPMCALLBACK_TRANS_PROGRESS: - pass - - elif what == rpm.RPMCALLBACK_TRANS_STOP: - pass - - elif what == rpm.RPMCALLBACK_INST_OPEN_FILE: - self.lastmsg = None - hdr = None - if h is not None: - try: - hdr, rpmloc = h - except: - rpmloc = h - hdr = readRpmHeader(self.ts, h) - - handle = self._makeHandle(hdr) - fd = os.open(rpmloc, os.O_RDONLY) - self.callbackfilehandles[handle]=fd - if hdr['name'] not in self.installed_pkg_names: - self.installed_pkg_names.append(hdr['name']) - self.total_installed += 1 - return fd - else: - self._localprint("No header - huh?") - - elif what == rpm.RPMCALLBACK_INST_CLOSE_FILE: - hdr = None - if h is not None: - try: - hdr, rpmloc = h - except: - rpmloc = h - hdr = readRpmHeader(self.ts, h) - - handle = self._makeHandle(hdr) - os.close(self.callbackfilehandles[handle]) - fd = 0 - - # log stuff - #pkgtup = self._dopkgtup(hdr) - self.logString.append(self._logPkgString(hdr)) - - elif what == rpm.RPMCALLBACK_INST_PROGRESS: - if h is not None: - percent = (self.total_installed*100L)/self.total_actions - if total > 0: - try: - hdr, rpmloc = h - except: - rpmloc = h - - m = re.match("(.*)-(\d+.*)-(\d+\.\d+)\.(.+)\.rpm", os.path.basename(rpmloc)) - if m: - pkgname = m.group(1) - else: - pkgname = os.path.basename(rpmloc) - if self.output: - fmt = self._makefmt(percent) - msg = fmt % (self.headmsg, pkgname) - if msg != self.lastmsg: - self.lastmsg = msg - - msger.info(msg) - - if self.total_installed == self.total_actions: - msger.raw('') - msger.verbose('\n'.join(self.logString)) - - elif what == rpm.RPMCALLBACK_UNINST_START: - pass - - elif what == rpm.RPMCALLBACK_UNINST_PROGRESS: - pass - - elif what == rpm.RPMCALLBACK_UNINST_STOP: - self.total_removed += 1 - - elif what == rpm.RPMCALLBACK_REPACKAGE_START: - pass - - elif what == rpm.RPMCALLBACK_REPACKAGE_STOP: - pass - - elif what == rpm.RPMCALLBACK_REPACKAGE_PROGRESS: - pass - -def readRpmHeader(ts, filename): - """ Read an rpm header. """ - - fd = os.open(filename, os.O_RDONLY) - h = ts.hdrFromFdno(fd) - os.close(fd) - return h - -def splitFilename(filename): - """ Pass in a standard style rpm fullname - - Return a name, version, release, epoch, arch, e.g.:: - foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 - 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 - """ - - if filename[-4:] == '.rpm': - filename = filename[:-4] - - archIndex = filename.rfind('.') - arch = filename[archIndex+1:] - - relIndex = filename[:archIndex].rfind('-') - rel = filename[relIndex+1:archIndex] - - verIndex = filename[:relIndex].rfind('-') - ver = filename[verIndex+1:relIndex] - - epochIndex = filename.find(':') - if epochIndex == -1: - epoch = '' - else: - epoch = filename[:epochIndex] - - name = filename[epochIndex + 1:verIndex] - return name, ver, rel, epoch, arch - -def getCanonX86Arch(arch): - # - if arch == "i586": - f = open("/proc/cpuinfo", "r") - lines = f.readlines() - f.close() - for line in lines: - if line.startswith("model name") and line.find("Geode(TM)") != -1: - return "geode" - return arch - # only athlon vs i686 isn't handled with uname currently - if arch != "i686": - return arch - - # if we're i686 and AuthenticAMD, then we should be an athlon - f = open("/proc/cpuinfo", "r") - lines = f.readlines() - f.close() - for line in lines: - if line.startswith("vendor") and line.find("AuthenticAMD") != -1: - return "athlon" - # i686 doesn't guarantee cmov, but we depend on it - elif line.startswith("flags") and line.find("cmov") == -1: - return "i586" - - return arch - -def getCanonX86_64Arch(arch): - if arch != "x86_64": - return arch - - vendor = None - f = open("/proc/cpuinfo", "r") - lines = f.readlines() - f.close() - for line in lines: - if line.startswith("vendor_id"): - vendor = line.split(':')[1] - break - if vendor is None: - return arch - - if vendor.find("Authentic AMD") != -1 or vendor.find("AuthenticAMD") != -1: - return "amd64" - if vendor.find("GenuineIntel") != -1: - return "ia32e" - return arch - -def getCanonArch(): - arch = os.uname()[4] - - if (len(arch) == 4 and arch[0] == "i" and arch[2:4] == "86"): - return getCanonX86Arch(arch) - - if arch == "x86_64": - return getCanonX86_64Arch(arch) - - return arch - -# Copy from libsatsolver:poolarch.c, with cleanup -archPolicies = { - "x86_64": "x86_64:i686:i586:i486:i386", - "i686": "i686:i586:i486:i386", - "i586": "i586:i486:i386", - "ia64": "ia64:i686:i586:i486:i386", - "armv7tnhl": "armv7tnhl:armv7thl:armv7nhl:armv7hl", - "armv7thl": "armv7thl:armv7hl", - "armv7nhl": "armv7nhl:armv7hl", - "armv7hl": "armv7hl", - "armv7l": "armv7l:armv6l:armv5tejl:armv5tel:armv5l:armv4tl:armv4l:armv3l", - "armv6l": "armv6l:armv5tejl:armv5tel:armv5l:armv4tl:armv4l:armv3l", - "armv5tejl": "armv5tejl:armv5tel:armv5l:armv4tl:armv4l:armv3l", - "armv5tel": "armv5tel:armv5l:armv4tl:armv4l:armv3l", - "armv5l": "armv5l:armv4tl:armv4l:armv3l", -} - -# dict mapping arch -> ( multicompat, best personality, biarch personality ) -multilibArches = { - "x86_64": ( "athlon", "x86_64", "athlon" ), -} - -# from yumUtils.py -arches = { - # ia32 - "athlon": "i686", - "i686": "i586", - "geode": "i586", - "i586": "i486", - "i486": "i386", - "i386": "noarch", - - # amd64 - "x86_64": "athlon", - "amd64": "x86_64", - "ia32e": "x86_64", - - # arm - "armv7tnhl": "armv7nhl", - "armv7nhl": "armv7hl", - "armv7hl": "noarch", - "armv7l": "armv6l", - "armv6l": "armv5tejl", - "armv5tejl": "armv5tel", - "armv5tel": "noarch", - - #itanium - "ia64": "noarch", -} - -def isMultiLibArch(arch=None): - """returns true if arch is a multilib arch, false if not""" - if arch is None: - arch = getCanonArch() - - if not arches.has_key(arch): # or we could check if it is noarch - return False - - if multilibArches.has_key(arch): - return True - - if multilibArches.has_key(arches[arch]): - return True - - return False - -def getBaseArch(): - myarch = getCanonArch() - if not arches.has_key(myarch): - return myarch - - if isMultiLibArch(arch=myarch): - if multilibArches.has_key(myarch): - return myarch - else: - return arches[myarch] - - if arches.has_key(myarch): - basearch = myarch - value = arches[basearch] - while value != 'noarch': - basearch = value - value = arches[basearch] - - return basearch - -def checkRpmIntegrity(bin_rpm, package): - return runner.quiet([bin_rpm, "-K", "--nosignature", package]) - -def checkSig(ts, package): - """ Takes a transaction set and a package, check it's sigs, - return 0 if they are all fine - return 1 if the gpg key can't be found - return 2 if the header is in someway damaged - return 3 if the key is not trusted - return 4 if the pkg is not gpg or pgp signed - """ - - value = 0 - currentflags = ts.setVSFlags(0) - fdno = os.open(package, os.O_RDONLY) - try: - hdr = ts.hdrFromFdno(fdno) - - except rpm.error, e: - if str(e) == "public key not availaiable": - value = 1 - if str(e) == "public key not available": - value = 1 - if str(e) == "public key not trusted": - value = 3 - if str(e) == "error reading package header": - value = 2 - else: - error, siginfo = getSigInfo(hdr) - if error == 101: - os.close(fdno) - del hdr - value = 4 - else: - del hdr - - try: - os.close(fdno) - except OSError: - pass - - ts.setVSFlags(currentflags) # put things back like they were before - return value - -def getSigInfo(hdr): - """ checks signature from an hdr hand back signature information and/or - an error code - """ - - import locale - locale.setlocale(locale.LC_ALL, 'C') - - string = '%|DSAHEADER?{%{DSAHEADER:pgpsig}}:{%|RSAHEADER?{%{RSAHEADER:pgpsig}}:{%|SIGGPG?{%{SIGGPG:pgpsig}}:{%|SIGPGP?{%{SIGPGP:pgpsig}}:{(none)}|}|}|}|' - siginfo = hdr.sprintf(string) - if siginfo != '(none)': - error = 0 - sigtype, sigdate, sigid = siginfo.split(',') - else: - error = 101 - sigtype = 'MD5' - sigdate = 'None' - sigid = 'None' - - infotuple = (sigtype, sigdate, sigid) - return error, infotuple - -def checkRepositoryEULA(name, repo): - """ This function is to check the EULA file if provided. - return True: no EULA or accepted - return False: user declined the EULA - """ - - import tempfile - import shutil - import urlparse - import urllib2 as u2 - import httplib - from mic.utils.errors import CreatorError - - def _check_and_download_url(u2opener, url, savepath): - try: - if u2opener: - f = u2opener.open(url) - else: - f = u2.urlopen(url) - except u2.HTTPError, httperror: - if httperror.code in (404, 503): - return None - else: - raise CreatorError(httperror) - except OSError, oserr: - if oserr.errno == 2: - return None - else: - raise CreatorError(oserr) - except IOError, oserr: - if hasattr(oserr, "reason") and oserr.reason.errno == 2: - return None - else: - raise CreatorError(oserr) - except u2.URLError, err: - raise CreatorError(err) - except httplib.HTTPException, e: - raise CreatorError(e) - - # save to file - licf = open(savepath, "w") - licf.write(f.read()) - licf.close() - f.close() - - return savepath - - def _pager_file(savepath): - - if os.path.splitext(savepath)[1].upper() in ('.HTM', '.HTML'): - pagers = ('w3m', 'links', 'lynx', 'less', 'more') - else: - pagers = ('less', 'more') - - file_showed = False - for pager in pagers: - cmd = "%s %s" % (pager, savepath) - try: - os.system(cmd) - except OSError: - continue - else: - file_showed = True - break - - if not file_showed: - f = open(savepath) - msger.raw(f.read()) - f.close() - msger.pause() - - # when proxy needed, make urllib2 follow it - proxy = repo.proxy - proxy_username = repo.proxy_username - proxy_password = repo.proxy_password - - if not proxy: - proxy = get_proxy_for(repo.baseurl[0]) - - handlers = [] - auth_handler = u2.HTTPBasicAuthHandler(u2.HTTPPasswordMgrWithDefaultRealm()) - u2opener = None - if proxy: - if proxy_username: - proxy_netloc = urlparse.urlsplit(proxy).netloc - if proxy_password: - proxy_url = 'http://%s:%s@%s' % (proxy_username, proxy_password, proxy_netloc) - else: - proxy_url = 'http://%s@%s' % (proxy_username, proxy_netloc) - else: - proxy_url = proxy - - proxy_support = u2.ProxyHandler({'http': proxy_url, - 'https': proxy_url, - 'ftp': proxy_url}) - handlers.append(proxy_support) - - # download all remote files to one temp dir - baseurl = None - repo_lic_dir = tempfile.mkdtemp(prefix = 'repolic') - - for url in repo.baseurl: - tmphandlers = handlers[:] - - (scheme, host, path, parm, query, frag) = urlparse.urlparse(url.rstrip('/') + '/') - if scheme not in ("http", "https", "ftp", "ftps", "file"): - raise CreatorError("Error: invalid url %s" % url) - - if '@' in host: - try: - user_pass, host = host.split('@', 1) - if ':' in user_pass: - user, password = user_pass.split(':', 1) - except ValueError, e: - raise CreatorError('Bad URL: %s' % url) - - msger.verbose("adding HTTP auth: %s, XXXXXXXX" %(user)) - auth_handler.add_password(None, host, user, password) - tmphandlers.append(auth_handler) - url = scheme + "://" + host + path + parm + query + frag - - if tmphandlers: - u2opener = u2.build_opener(*tmphandlers) - - # try to download - repo_eula_url = urlparse.urljoin(url, "LICENSE.txt") - repo_eula_path = _check_and_download_url( - u2opener, - repo_eula_url, - os.path.join(repo_lic_dir, repo.id + '_LICENSE.txt')) - if repo_eula_path: - # found - baseurl = url - break - - if not baseurl: - shutil.rmtree(repo_lic_dir) #cleanup - return True - - # show the license file - msger.info('For the software packages in this yum repo:') - msger.info(' %s: %s' % (name, baseurl)) - msger.info('There is an "End User License Agreement" file that need to be checked.') - msger.info('Please read the terms and conditions outlined in it and answer the followed qustions.') - msger.pause() - - _pager_file(repo_eula_path) - - # Asking for the "Accept/Decline" - if not msger.ask('Would you agree to the terms and conditions outlined in the above End User License Agreement?'): - msger.warning('Will not install pkgs from this repo.') - shutil.rmtree(repo_lic_dir) #cleanup - return False - - # try to find support_info.html for extra infomation - repo_info_url = urlparse.urljoin(baseurl, "support_info.html") - repo_info_path = _check_and_download_url( - u2opener, - repo_info_url, - os.path.join(repo_lic_dir, repo.id + '_support_info.html')) - if repo_info_path: - msger.info('There is one more file in the repo for additional support information, please read it') - msger.pause() - _pager_file(repo_info_path) - - #cleanup - shutil.rmtree(repo_lic_dir) - return True diff --git a/scripts/lib/mic/utils/runner.py b/scripts/lib/mic/utils/runner.py deleted file mode 100644 index fded3c93fa..0000000000 --- a/scripts/lib/mic/utils/runner.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import subprocess - -from mic import msger - -def runtool(cmdln_or_args, catch=1): - """ wrapper for most of the subprocess calls - input: - cmdln_or_args: can be both args and cmdln str (shell=True) - catch: 0, quitely run - 1, only STDOUT - 2, only STDERR - 3, both STDOUT and STDERR - return: - (rc, output) - if catch==0: the output will always None - """ - - if catch not in (0, 1, 2, 3): - # invalid catch selection, will cause exception, that's good - return None - - if isinstance(cmdln_or_args, list): - cmd = cmdln_or_args[0] - shell = False - else: - import shlex - cmd = shlex.split(cmdln_or_args)[0] - shell = True - - if catch != 3: - dev_null = os.open("/dev/null", os.O_WRONLY) - - if catch == 0: - sout = dev_null - serr = dev_null - elif catch == 1: - sout = subprocess.PIPE - serr = dev_null - elif catch == 2: - sout = dev_null - serr = subprocess.PIPE - elif catch == 3: - sout = subprocess.PIPE - serr = subprocess.STDOUT - - try: - p = subprocess.Popen(cmdln_or_args, stdout=sout, - stderr=serr, shell=shell) - (sout, serr) = p.communicate() - # combine stdout and stderr, filter None out - out = ''.join(filter(None, [sout, serr])) - except OSError, e: - if e.errno == 2: - # [Errno 2] No such file or directory - msger.error('Cannot run command: %s, lost dependency?' % cmd) - else: - raise # relay - finally: - if catch != 3: - os.close(dev_null) - - return (p.returncode, out) - -def show(cmdln_or_args): - # show all the message using msger.verbose - - rc, out = runtool(cmdln_or_args, catch=3) - - if isinstance(cmdln_or_args, list): - cmd = ' '.join(cmdln_or_args) - else: - cmd = cmdln_or_args - - msg = 'running command: "%s"' % cmd - if out: out = out.strip() - if out: - msg += ', with output::' - msg += '\n +----------------' - for line in out.splitlines(): - msg += '\n | %s' % line - msg += '\n +----------------' - - msger.verbose(msg) - return rc - -def outs(cmdln_or_args, catch=1): - # get the outputs of tools - return runtool(cmdln_or_args, catch)[1].strip() - -def quiet(cmdln_or_args): - return runtool(cmdln_or_args, catch=0)[0] |