From 00dcdb29c89634ab267d328eb00f8eb70c696655 Mon Sep 17 00:00:00 2001 From: Tom Zanussi Date: Mon, 4 Aug 2014 07:55:07 -0500 Subject: wic: Remove 3rdparty/urlgrabber wic doesn't use it, so remove it. Signed-off-by: Tom Zanussi --- .../3rdparty/pykickstart/urlgrabber/__init__.py | 53 - .../3rdparty/pykickstart/urlgrabber/byterange.py | 463 ------ .../mic/3rdparty/pykickstart/urlgrabber/grabber.py | 1477 -------------------- .../3rdparty/pykickstart/urlgrabber/keepalive.py | 617 -------- .../mic/3rdparty/pykickstart/urlgrabber/mirror.py | 458 ------ .../3rdparty/pykickstart/urlgrabber/progress.py | 530 ------- .../3rdparty/pykickstart/urlgrabber/sslfactory.py | 90 -- 7 files changed, 3688 deletions(-) delete mode 100644 scripts/lib/mic/3rdparty/pykickstart/urlgrabber/__init__.py delete mode 100644 scripts/lib/mic/3rdparty/pykickstart/urlgrabber/byterange.py delete mode 100644 scripts/lib/mic/3rdparty/pykickstart/urlgrabber/grabber.py delete mode 100644 scripts/lib/mic/3rdparty/pykickstart/urlgrabber/keepalive.py delete mode 100644 scripts/lib/mic/3rdparty/pykickstart/urlgrabber/mirror.py delete mode 100644 scripts/lib/mic/3rdparty/pykickstart/urlgrabber/progress.py delete mode 100644 scripts/lib/mic/3rdparty/pykickstart/urlgrabber/sslfactory.py (limited to 'scripts/lib') diff --git a/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/__init__.py b/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/__init__.py deleted file mode 100644 index 7bcd9d5541..0000000000 --- a/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -# Copyright 2002-2006 Michael D. Stenner, Ryan Tomayko - -# $Id: __init__.py,v 1.20 2006/09/22 00:58:55 mstenner Exp $ - -"""A high-level cross-protocol url-grabber. - -Using urlgrabber, data can be fetched in three basic ways: - - urlgrab(url) copy the file to the local filesystem - urlopen(url) open the remote file and return a file object - (like urllib2.urlopen) - urlread(url) return the contents of the file as a string - -When using these functions (or methods), urlgrabber supports the -following features: - - * identical behavior for http://, ftp://, and file:// urls - * http keepalive - faster downloads of many files by using - only a single connection - * byte ranges - fetch only a portion of the file - * reget - for a urlgrab, resume a partial download - * progress meters - the ability to report download progress - automatically, even when using urlopen! - * throttling - restrict bandwidth usage - * retries - automatically retry a download if it fails. The - number of retries and failure types are configurable. - * authenticated server access for http and ftp - * proxy support - support for authenticated http and ftp proxies - * mirror groups - treat a list of mirrors as a single source, - automatically switching mirrors if there is a failure. -""" - -__version__ = '3.1.0' -__date__ = '2006/09/21' -__author__ = 'Michael D. Stenner , ' \ - 'Ryan Tomayko ' -__url__ = 'http://linux.duke.edu/projects/urlgrabber/' - -from grabber import urlgrab, urlopen, urlread diff --git a/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/byterange.py b/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/byterange.py deleted file mode 100644 index 001b4e32d6..0000000000 --- a/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/byterange.py +++ /dev/null @@ -1,463 +0,0 @@ -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, -# Boston, MA 02111-1307 USA - -# This file is part of urlgrabber, a high-level cross-protocol url-grabber -# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko - -# $Id: byterange.py,v 1.12 2006/07/20 20:15:58 mstenner Exp $ - -import os -import stat -import urllib -import urllib2 -import rfc822 - -DEBUG = None - -try: - from cStringIO import StringIO -except ImportError, msg: - from StringIO import StringIO - -class RangeError(IOError): - """Error raised when an unsatisfiable range is requested.""" - pass - -class HTTPRangeHandler(urllib2.BaseHandler): - """Handler that enables HTTP Range headers. - - This was extremely simple. The Range header is a HTTP feature to - begin with so all this class does is tell urllib2 that the - "206 Partial Content" reponse from the HTTP server is what we - expected. - - Example: - import urllib2 - import byterange - - range_handler = range.HTTPRangeHandler() - opener = urllib2.build_opener(range_handler) - - # install it - urllib2.install_opener(opener) - - # create Request and set Range header - req = urllib2.Request('http://www.python.org/') - req.header['Range'] = 'bytes=30-50' - f = urllib2.urlopen(req) - """ - - def http_error_206(self, req, fp, code, msg, hdrs): - # 206 Partial Content Response - r = urllib.addinfourl(fp, hdrs, req.get_full_url()) - r.code = code - r.msg = msg - return r - - def http_error_416(self, req, fp, code, msg, hdrs): - # HTTP's Range Not Satisfiable error - raise RangeError('Requested Range Not Satisfiable') - -class HTTPSRangeHandler(HTTPRangeHandler): - """ Range Header support for HTTPS. """ - - def https_error_206(self, req, fp, code, msg, hdrs): - return self.http_error_206(req, fp, code, msg, hdrs) - - def https_error_416(self, req, fp, code, msg, hdrs): - self.https_error_416(req, fp, code, msg, hdrs) - -class RangeableFileObject: - """File object wrapper to enable raw range handling. - This was implemented primarilary for handling range - specifications for file:// urls. This object effectively makes - a file object look like it consists only of a range of bytes in - the stream. - - Examples: - # expose 10 bytes, starting at byte position 20, from - # /etc/aliases. - >>> fo = RangeableFileObject(file('/etc/passwd', 'r'), (20,30)) - # seek seeks within the range (to position 23 in this case) - >>> fo.seek(3) - # tell tells where your at _within the range_ (position 3 in - # this case) - >>> fo.tell() - # read EOFs if an attempt is made to read past the last - # byte in the range. the following will return only 7 bytes. - >>> fo.read(30) - """ - - def __init__(self, fo, rangetup): - """Create a RangeableFileObject. - fo -- a file like object. only the read() method need be - supported but supporting an optimized seek() is - preferable. - rangetup -- a (firstbyte,lastbyte) tuple specifying the range - to work over. - The file object provided is assumed to be at byte offset 0. - """ - self.fo = fo - (self.firstbyte, self.lastbyte) = range_tuple_normalize(rangetup) - self.realpos = 0 - self._do_seek(self.firstbyte) - - def __getattr__(self, name): - """This effectively allows us to wrap at the instance level. - Any attribute not found in _this_ object will be searched for - in self.fo. This includes methods.""" - if hasattr(self.fo, name): - return getattr(self.fo, name) - raise AttributeError, name - - def tell(self): - """Return the position within the range. - This is different from fo.seek in that position 0 is the - first byte position of the range tuple. For example, if - this object was created with a range tuple of (500,899), - tell() will return 0 when at byte position 500 of the file. - """ - return (self.realpos - self.firstbyte) - - def seek(self,offset,whence=0): - """Seek within the byte range. - Positioning is identical to that described under tell(). - """ - assert whence in (0, 1, 2) - if whence == 0: # absolute seek - realoffset = self.firstbyte + offset - elif whence == 1: # relative seek - realoffset = self.realpos + offset - elif whence == 2: # absolute from end of file - # XXX: are we raising the right Error here? - raise IOError('seek from end of file not supported.') - - # do not allow seek past lastbyte in range - if self.lastbyte and (realoffset >= self.lastbyte): - realoffset = self.lastbyte - - self._do_seek(realoffset - self.realpos) - - def read(self, size=-1): - """Read within the range. - This method will limit the size read based on the range. - """ - size = self._calc_read_size(size) - rslt = self.fo.read(size) - self.realpos += len(rslt) - return rslt - - def readline(self, size=-1): - """Read lines within the range. - This method will limit the size read based on the range. - """ - size = self._calc_read_size(size) - rslt = self.fo.readline(size) - self.realpos += len(rslt) - return rslt - - def _calc_read_size(self, size): - """Handles calculating the amount of data to read based on - the range. - """ - if self.lastbyte: - if size > -1: - if ((self.realpos + size) >= self.lastbyte): - size = (self.lastbyte - self.realpos) - else: - size = (self.lastbyte - self.realpos) - return size - - def _do_seek(self,offset): - """Seek based on whether wrapped object supports seek(). - offset is relative to the current position (self.realpos). - """ - assert offset >= 0 - if not hasattr(self.fo, 'seek'): - self._poor_mans_seek(offset) - else: - self.fo.seek(self.realpos + offset) - self.realpos+= offset - - def _poor_mans_seek(self,offset): - """Seek by calling the wrapped file objects read() method. - This is used for file like objects that do not have native - seek support. The wrapped objects read() method is called - to manually seek to the desired position. - offset -- read this number of bytes from the wrapped - file object. - raise RangeError if we encounter EOF before reaching the - specified offset. - """ - pos = 0 - bufsize = 1024 - while pos < offset: - if (pos + bufsize) > offset: - bufsize = offset - pos - buf = self.fo.read(bufsize) - if len(buf) != bufsize: - raise RangeError('Requested Range Not Satisfiable') - pos+= bufsize - -class FileRangeHandler(urllib2.FileHandler): - """FileHandler subclass that adds Range support. - This class handles Range headers exactly like an HTTP - server would. - """ - def open_local_file(self, req): - import mimetypes - import mimetools - host = req.get_host() - file = req.get_selector() - localfile = urllib.url2pathname(file) - stats = os.stat(localfile) - size = stats[stat.ST_SIZE] - modified = rfc822.formatdate(stats[stat.ST_MTIME]) - mtype = mimetypes.guess_type(file)[0] - if host: - host, port = urllib.splitport(host) - if port or socket.gethostbyname(host) not in self.get_names(): - raise urllib2.URLError('file not on local host') - fo = open(localfile,'rb') - brange = req.headers.get('Range',None) - brange = range_header_to_tuple(brange) - assert brange != () - if brange: - (fb,lb) = brange - if lb == '': lb = size - if fb < 0 or fb > size or lb > size: - raise RangeError('Requested Range Not Satisfiable') - size = (lb - fb) - fo = RangeableFileObject(fo, (fb,lb)) - headers = mimetools.Message(StringIO( - 'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' % - (mtype or 'text/plain', size, modified))) - return urllib.addinfourl(fo, headers, 'file:'+file) - - -# FTP Range Support -# Unfortunately, a large amount of base FTP code had to be copied -# from urllib and urllib2 in order to insert the FTP REST command. -# Code modifications for range support have been commented as -# follows: -# -- range support modifications start/end here - -from urllib import splitport, splituser, splitpasswd, splitattr, \ - unquote, addclosehook, addinfourl -import ftplib -import socket -import sys -import ftplib -import mimetypes -import mimetools - -class FTPRangeHandler(urllib2.FTPHandler): - def ftp_open(self, req): - host = req.get_host() - if not host: - raise IOError, ('ftp error', 'no host given') - host, port = splitport(host) - if port is None: - port = ftplib.FTP_PORT - - # username/password handling - user, host = splituser(host) - if user: - user, passwd = splitpasswd(user) - else: - passwd = None - host = unquote(host) - user = unquote(user or '') - passwd = unquote(passwd or '') - - try: - host = socket.gethostbyname(host) - except socket.error, msg: - raise urllib2.URLError(msg) - path, attrs = splitattr(req.get_selector()) - dirs = path.split('/') - dirs = map(unquote, dirs) - dirs, file = dirs[:-1], dirs[-1] - if dirs and not dirs[0]: - dirs = dirs[1:] - try: - fw = self.connect_ftp(user, passwd, host, port, dirs) - type = file and 'I' or 'D' - for attr in attrs: - attr, value = splitattr(attr) - if attr.lower() == 'type' and \ - value in ('a', 'A', 'i', 'I', 'd', 'D'): - type = value.upper() - - # -- range support modifications start here - rest = None - range_tup = range_header_to_tuple(req.headers.get('Range',None)) - assert range_tup != () - if range_tup: - (fb,lb) = range_tup - if fb > 0: rest = fb - # -- range support modifications end here - - fp, retrlen = fw.retrfile(file, type, rest) - - # -- range support modifications start here - if range_tup: - (fb,lb) = range_tup - if lb == '': - if retrlen is None or retrlen == 0: - raise RangeError('Requested Range Not Satisfiable due to unobtainable file length.') - lb = retrlen - retrlen = lb - fb - if retrlen < 0: - # beginning of range is larger than file - raise RangeError('Requested Range Not Satisfiable') - else: - retrlen = lb - fb - fp = RangeableFileObject(fp, (0,retrlen)) - # -- range support modifications end here - - headers = "" - mtype = mimetypes.guess_type(req.get_full_url())[0] - if mtype: - headers += "Content-Type: %s\n" % mtype - if retrlen is not None and retrlen >= 0: - headers += "Content-Length: %d\n" % retrlen - sf = StringIO(headers) - headers = mimetools.Message(sf) - return addinfourl(fp, headers, req.get_full_url()) - except ftplib.all_errors, msg: - raise IOError, ('ftp error', msg), sys.exc_info()[2] - - def connect_ftp(self, user, passwd, host, port, dirs): - fw = ftpwrapper(user, passwd, host, port, dirs) - return fw - -class ftpwrapper(urllib.ftpwrapper): - # range support note: - # this ftpwrapper code is copied directly from - # urllib. The only enhancement is to add the rest - # argument and pass it on to ftp.ntransfercmd - def retrfile(self, file, type, rest=None): - self.endtransfer() - if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1 - else: cmd = 'TYPE ' + type; isdir = 0 - try: - self.ftp.voidcmd(cmd) - except ftplib.all_errors: - self.init() - self.ftp.voidcmd(cmd) - conn = None - if file and not isdir: - # Use nlst to see if the file exists at all - try: - self.ftp.nlst(file) - except ftplib.error_perm, reason: - raise IOError, ('ftp error', reason), sys.exc_info()[2] - # Restore the transfer mode! - self.ftp.voidcmd(cmd) - # Try to retrieve as a file - try: - cmd = 'RETR ' + file - conn = self.ftp.ntransfercmd(cmd, rest) - except ftplib.error_perm, reason: - if str(reason)[:3] == '501': - # workaround for REST not supported error - fp, retrlen = self.retrfile(file, type) - fp = RangeableFileObject(fp, (rest,'')) - return (fp, retrlen) - elif str(reason)[:3] != '550': - raise IOError, ('ftp error', reason), sys.exc_info()[2] - if not conn: - # Set transfer mode to ASCII! - self.ftp.voidcmd('TYPE A') - # Try a directory listing - if file: cmd = 'LIST ' + file - else: cmd = 'LIST' - conn = self.ftp.ntransfercmd(cmd) - self.busy = 1 - # Pass back both a suitably decorated object and a retrieval length - return (addclosehook(conn[0].makefile('rb'), - self.endtransfer), conn[1]) - - -#################################################################### -# Range Tuple Functions -# XXX: These range tuple functions might go better in a class. - -_rangere = None -def range_header_to_tuple(range_header): - """Get a (firstbyte,lastbyte) tuple from a Range header value. - - Range headers have the form "bytes=-". This - function pulls the firstbyte and lastbyte values and returns - a (firstbyte,lastbyte) tuple. If lastbyte is not specified in - the header value, it is returned as an empty string in the - tuple. - - Return None if range_header is None - Return () if range_header does not conform to the range spec - pattern. - - """ - global _rangere - if range_header is None: return None - if _rangere is None: - import re - _rangere = re.compile(r'^bytes=(\d{1,})-(\d*)') - match = _rangere.match(range_header) - if match: - tup = range_tuple_normalize(match.group(1,2)) - if tup and tup[1]: - tup = (tup[0],tup[1]+1) - return tup - return () - -def range_tuple_to_header(range_tup): - """Convert a range tuple to a Range header value. - Return a string of the form "bytes=-" or None - if no range is needed. - """ - if range_tup is None: return None - range_tup = range_tuple_normalize(range_tup) - if range_tup: - if range_tup[1]: - range_tup = (range_tup[0],range_tup[1] - 1) - return 'bytes=%s-%s' % range_tup - -def range_tuple_normalize(range_tup): - """Normalize a (first_byte,last_byte) range tuple. - Return a tuple whose first element is guaranteed to be an int - and whose second element will be '' (meaning: the last byte) or - an int. Finally, return None if the normalized tuple == (0,'') - as that is equivelant to retrieving the entire file. - """ - if range_tup is None: return None - # handle first byte - fb = range_tup[0] - if fb in (None,''): fb = 0 - else: fb = int(fb) - # handle last byte - try: lb = range_tup[1] - except IndexError: lb = '' - else: - if lb is None: lb = '' - elif lb != '': lb = int(lb) - # check if range is over the entire file - if (fb,lb) == (0,''): return None - # check that the range is valid - if lb < fb: raise RangeError('Invalid byte range: %s-%s' % (fb,lb)) - return (fb,lb) - diff --git a/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/grabber.py b/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/grabber.py deleted file mode 100644 index fefdab36f6..0000000000 --- a/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/grabber.py +++ /dev/null @@ -1,1477 +0,0 @@ -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, -# Boston, MA 02111-1307 USA - -# This file is part of urlgrabber, a high-level cross-protocol url-grabber -# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko - -"""A high-level cross-protocol url-grabber. - -GENERAL ARGUMENTS (kwargs) - - Where possible, the module-level default is indicated, and legal - values are provided. - - copy_local = 0 [0|1] - - ignored except for file:// urls, in which case it specifies - whether urlgrab should still make a copy of the file, or simply - point to the existing copy. The module level default for this - option is 0. - - close_connection = 0 [0|1] - - tells URLGrabber to close the connection after a file has been - transfered. This is ignored unless the download happens with the - http keepalive handler (keepalive=1). Otherwise, the connection - is left open for further use. The module level default for this - option is 0 (keepalive connections will not be closed). - - keepalive = 1 [0|1] - - specifies whether keepalive should be used for HTTP/1.1 servers - that support it. The module level default for this option is 1 - (keepalive is enabled). - - progress_obj = None - - a class instance that supports the following methods: - po.start(filename, url, basename, length, text) - # length will be None if unknown - po.update(read) # read == bytes read so far - po.end() - - text = None - - specifies an alternativ text item in the beginning of the progress - bar line. If not given, the basename of the file is used. - - throttle = 1.0 - - a number - if it's an int, it's the bytes/second throttle limit. - If it's a float, it is first multiplied by bandwidth. If throttle - == 0, throttling is disabled. If None, the module-level default - (which can be set on default_grabber.throttle) is used. See - BANDWIDTH THROTTLING for more information. - - timeout = None - - a positive float expressing the number of seconds to wait for socket - operations. If the value is None or 0.0, socket operations will block - forever. Setting this option causes urlgrabber to call the settimeout - method on the Socket object used for the request. See the Python - documentation on settimeout for more information. - http://www.python.org/doc/current/lib/socket-objects.html - - bandwidth = 0 - - the nominal max bandwidth in bytes/second. If throttle is a float - and bandwidth == 0, throttling is disabled. If None, the - module-level default (which can be set on - default_grabber.bandwidth) is used. See BANDWIDTH THROTTLING for - more information. - - range = None - - a tuple of the form (first_byte, last_byte) describing a byte - range to retrieve. Either or both of the values may set to - None. If first_byte is None, byte offset 0 is assumed. If - last_byte is None, the last byte available is assumed. Note that - the range specification is python-like in that (0,10) will yeild - the first 10 bytes of the file. - - If set to None, no range will be used. - - reget = None [None|'simple'|'check_timestamp'] - - whether to attempt to reget a partially-downloaded file. Reget - only applies to .urlgrab and (obviously) only if there is a - partially downloaded file. Reget has two modes: - - 'simple' -- the local file will always be trusted. If there - are 100 bytes in the local file, then the download will always - begin 100 bytes into the requested file. - - 'check_timestamp' -- the timestamp of the server file will be - compared to the timestamp of the local file. ONLY if the - local file is newer than or the same age as the server file - will reget be used. If the server file is newer, or the - timestamp is not returned, the entire file will be fetched. - - NOTE: urlgrabber can do very little to verify that the partial - file on disk is identical to the beginning of the remote file. - You may want to either employ a custom "checkfunc" or simply avoid - using reget in situations where corruption is a concern. - - user_agent = 'urlgrabber/VERSION' - - a string, usually of the form 'AGENT/VERSION' that is provided to - HTTP servers in the User-agent header. The module level default - for this option is "urlgrabber/VERSION". - - http_headers = None - - a tuple of 2-tuples, each containing a header and value. These - will be used for http and https requests only. For example, you - can do - http_headers = (('Pragma', 'no-cache'),) - - ftp_headers = None - - this is just like http_headers, but will be used for ftp requests. - - proxies = None - - a dictionary that maps protocol schemes to proxy hosts. For - example, to use a proxy server on host "foo" port 3128 for http - and https URLs: - proxies={ 'http' : 'http://foo:3128', 'https' : 'http://foo:3128' } - note that proxy authentication information may be provided using - normal URL constructs: - proxies={ 'http' : 'http://user:host@foo:3128' } - Lastly, if proxies is None, the default environment settings will - be used. - - prefix = None - - a url prefix that will be prepended to all requested urls. For - example: - g = URLGrabber(prefix='http://foo.com/mirror/') - g.urlgrab('some/file.txt') - ## this will fetch 'http://foo.com/mirror/some/file.txt' - This option exists primarily to allow identical behavior to - MirrorGroup (and derived) instances. Note: a '/' will be inserted - if necessary, so you cannot specify a prefix that ends with a - partial file or directory name. - - opener = None - - Overrides the default urllib2.OpenerDirector provided to urllib2 - when making requests. This option exists so that the urllib2 - handler chain may be customized. Note that the range, reget, - proxy, and keepalive features require that custom handlers be - provided to urllib2 in order to function properly. If an opener - option is provided, no attempt is made by urlgrabber to ensure - chain integrity. You are responsible for ensuring that any - extension handlers are present if said features are required. - - data = None - - Only relevant for the HTTP family (and ignored for other - protocols), this allows HTTP POSTs. When the data kwarg is - present (and not None), an HTTP request will automatically become - a POST rather than GET. This is done by direct passthrough to - urllib2. If you use this, you may also want to set the - 'Content-length' and 'Content-type' headers with the http_headers - option. Note that python 2.2 handles the case of these - badly and if you do not use the proper case (shown here), your - values will be overridden with the defaults. - - -RETRY RELATED ARGUMENTS - - retry = None - - the number of times to retry the grab before bailing. If this is - zero, it will retry forever. This was intentional... really, it - was :). If this value is not supplied or is supplied but is None - retrying does not occur. - - retrycodes = [-1,2,4,5,6,7] - - a sequence of errorcodes (values of e.errno) for which it should - retry. See the doc on URLGrabError for more details on this. You - might consider modifying a copy of the default codes rather than - building yours from scratch so that if the list is extended in the - future (or one code is split into two) you can still enjoy the - benefits of the default list. You can do that with something like - this: - - retrycodes = urlgrabber.grabber.URLGrabberOptions().retrycodes - if 12 not in retrycodes: - retrycodes.append(12) - - checkfunc = None - - a function to do additional checks. This defaults to None, which - means no additional checking. The function should simply return - on a successful check. It should raise URLGrabError on an - unsuccessful check. Raising of any other exception will be - considered immediate failure and no retries will occur. - - If it raises URLGrabError, the error code will determine the retry - behavior. Negative error numbers are reserved for use by these - passed in functions, so you can use many negative numbers for - different types of failure. By default, -1 results in a retry, - but this can be customized with retrycodes. - - If you simply pass in a function, it will be given exactly one - argument: a CallbackObject instance with the .url attribute - defined and either .filename (for urlgrab) or .data (for urlread). - For urlgrab, .filename is the name of the local file. For - urlread, .data is the actual string data. If you need other - arguments passed to the callback (program state of some sort), you - can do so like this: - - checkfunc=(function, ('arg1', 2), {'kwarg': 3}) - - if the downloaded file has filename /tmp/stuff, then this will - result in this call (for urlgrab): - - function(obj, 'arg1', 2, kwarg=3) - # obj.filename = '/tmp/stuff' - # obj.url = 'http://foo.com/stuff' - - NOTE: both the "args" tuple and "kwargs" dict must be present if - you use this syntax, but either (or both) can be empty. - - failure_callback = None - - The callback that gets called during retries when an attempt to - fetch a file fails. The syntax for specifying the callback is - identical to checkfunc, except for the attributes defined in the - CallbackObject instance. The attributes for failure_callback are: - - exception = the raised exception - url = the url we're trying to fetch - tries = the number of tries so far (including this one) - retry = the value of the retry option - - The callback is present primarily to inform the calling program of - the failure, but if it raises an exception (including the one it's - passed) that exception will NOT be caught and will therefore cause - future retries to be aborted. - - The callback is called for EVERY failure, including the last one. - On the last try, the callback can raise an alternate exception, - but it cannot (without severe trickiness) prevent the exception - from being raised. - - interrupt_callback = None - - This callback is called if KeyboardInterrupt is received at any - point in the transfer. Basically, this callback can have three - impacts on the fetch process based on the way it exits: - - 1) raise no exception: the current fetch will be aborted, but - any further retries will still take place - - 2) raise a URLGrabError: if you're using a MirrorGroup, then - this will prompt a failover to the next mirror according to - the behavior of the MirrorGroup subclass. It is recommended - that you raise URLGrabError with code 15, 'user abort'. If - you are NOT using a MirrorGroup subclass, then this is the - same as (3). - - 3) raise some other exception (such as KeyboardInterrupt), which - will not be caught at either the grabber or mirror levels. - That is, it will be raised up all the way to the caller. - - This callback is very similar to failure_callback. They are - passed the same arguments, so you could use the same function for - both. - - urlparser = URLParser() - - The URLParser class handles pre-processing of URLs, including - auth-handling for user/pass encoded in http urls, file handing - (that is, filenames not sent as a URL), and URL quoting. If you - want to override any of this behavior, you can pass in a - replacement instance. See also the 'quote' option. - - quote = None - - Whether or not to quote the path portion of a url. - quote = 1 -> quote the URLs (they're not quoted yet) - quote = 0 -> do not quote them (they're already quoted) - quote = None -> guess what to do - - This option only affects proper urls like 'file:///etc/passwd'; it - does not affect 'raw' filenames like '/etc/passwd'. The latter - will always be quoted as they are converted to URLs. Also, only - the path part of a url is quoted. If you need more fine-grained - control, you should probably subclass URLParser and pass it in via - the 'urlparser' option. - -BANDWIDTH THROTTLING - - urlgrabber supports throttling via two values: throttle and - bandwidth Between the two, you can either specify and absolute - throttle threshold or specify a theshold as a fraction of maximum - available bandwidth. - - throttle is a number - if it's an int, it's the bytes/second - throttle limit. If it's a float, it is first multiplied by - bandwidth. If throttle == 0, throttling is disabled. If None, the - module-level default (which can be set with set_throttle) is used. - - bandwidth is the nominal max bandwidth in bytes/second. If throttle - is a float and bandwidth == 0, throttling is disabled. If None, the - module-level default (which can be set with set_bandwidth) is used. - - THROTTLING EXAMPLES: - - Lets say you have a 100 Mbps connection. This is (about) 10^8 bits - per second, or 12,500,000 Bytes per second. You have a number of - throttling options: - - *) set_bandwidth(12500000); set_throttle(0.5) # throttle is a float - - This will limit urlgrab to use half of your available bandwidth. - - *) set_throttle(6250000) # throttle is an int - - This will also limit urlgrab to use half of your available - bandwidth, regardless of what bandwidth is set to. - - *) set_throttle(6250000); set_throttle(1.0) # float - - Use half your bandwidth - - *) set_throttle(6250000); set_throttle(2.0) # float - - Use up to 12,500,000 Bytes per second (your nominal max bandwidth) - - *) set_throttle(6250000); set_throttle(0) # throttle = 0 - - Disable throttling - this is more efficient than a very large - throttle setting. - - *) set_throttle(0); set_throttle(1.0) # throttle is float, bandwidth = 0 - - Disable throttling - this is the default when the module is loaded. - - SUGGESTED AUTHOR IMPLEMENTATION (THROTTLING) - - While this is flexible, it's not extremely obvious to the user. I - suggest you implement a float throttle as a percent to make the - distinction between absolute and relative throttling very explicit. - - Also, you may want to convert the units to something more convenient - than bytes/second, such as kbps or kB/s, etc. - -""" - -# $Id: grabber.py,v 1.48 2006/09/22 00:58:05 mstenner Exp $ - -import os -import os.path -import sys -import urlparse -import rfc822 -import time -import string -import urllib -import urllib2 -from stat import * # S_* and ST_* - -######################################################################## -# MODULE INITIALIZATION -######################################################################## -try: - exec('from ' + (__name__.split('.'))[0] + ' import __version__') -except: - __version__ = '???' - -import sslfactory - -auth_handler = urllib2.HTTPBasicAuthHandler( \ - urllib2.HTTPPasswordMgrWithDefaultRealm()) - -try: - from i18n import _ -except ImportError, msg: - def _(st): return st - -try: - from httplib import HTTPException -except ImportError, msg: - HTTPException = None - -try: - # This is a convenient way to make keepalive optional. - # Just rename the module so it can't be imported. - import keepalive - from keepalive import HTTPHandler, HTTPSHandler - have_keepalive = True -except ImportError, msg: - have_keepalive = False - -try: - # add in range support conditionally too - import byterange - from byterange import HTTPRangeHandler, HTTPSRangeHandler, \ - FileRangeHandler, FTPRangeHandler, range_tuple_normalize, \ - range_tuple_to_header, RangeError -except ImportError, msg: - range_handlers = () - RangeError = None - have_range = 0 -else: - range_handlers = (HTTPRangeHandler(), HTTPSRangeHandler(), - FileRangeHandler(), FTPRangeHandler()) - have_range = 1 - - -# check whether socket timeout support is available (Python >= 2.3) -import socket -try: - TimeoutError = socket.timeout - have_socket_timeout = True -except AttributeError: - TimeoutError = None - have_socket_timeout = False - -######################################################################## -# functions for debugging output. These functions are here because they -# are also part of the module initialization. -DEBUG = None -def set_logger(DBOBJ): - """Set the DEBUG object. This is called by _init_default_logger when - the environment variable URLGRABBER_DEBUG is set, but can also be - called by a calling program. Basically, if the calling program uses - the logging module and would like to incorporate urlgrabber logging, - then it can do so this way. It's probably not necessary as most - internal logging is only for debugging purposes. - - The passed-in object should be a logging.Logger instance. It will - be pushed into the keepalive and byterange modules if they're - being used. The mirror module pulls this object in on import, so - you will need to manually push into it. In fact, you may find it - tidier to simply push your logging object (or objects) into each - of these modules independently. - """ - - global DEBUG - DEBUG = DBOBJ - if have_keepalive and keepalive.DEBUG is None: - keepalive.DEBUG = DBOBJ - if have_range and byterange.DEBUG is None: - byterange.DEBUG = DBOBJ - if sslfactory.DEBUG is None: - sslfactory.DEBUG = DBOBJ - -def _init_default_logger(): - '''Examines the environment variable URLGRABBER_DEBUG and creates - a logging object (logging.logger) based on the contents. It takes - the form - - URLGRABBER_DEBUG=level,filename - - where "level" can be either an integer or a log level from the - logging module (DEBUG, INFO, etc). If the integer is zero or - less, logging will be disabled. Filename is the filename where - logs will be sent. If it is "-", then stdout will be used. If - the filename is empty or missing, stderr will be used. If the - variable cannot be processed or the logging module cannot be - imported (python < 2.3) then logging will be disabled. Here are - some examples: - - URLGRABBER_DEBUG=1,debug.txt # log everything to debug.txt - URLGRABBER_DEBUG=WARNING,- # log warning and higher to stdout - URLGRABBER_DEBUG=INFO # log info and higher to stderr - - This funtion is called during module initialization. It is not - intended to be called from outside. The only reason it is a - function at all is to keep the module-level namespace tidy and to - collect the code into a nice block.''' - - try: - dbinfo = os.environ['URLGRABBER_DEBUG'].split(',') - import logging - level = logging._levelNames.get(dbinfo[0], int(dbinfo[0])) - if level < 1: raise ValueError() - - formatter = logging.Formatter('%(asctime)s %(message)s') - if len(dbinfo) > 1: filename = dbinfo[1] - else: filename = '' - if filename == '': handler = logging.StreamHandler(sys.stderr) - elif filename == '-': handler = logging.StreamHandler(sys.stdout) - else: handler = logging.FileHandler(filename) - handler.setFormatter(formatter) - DBOBJ = logging.getLogger('urlgrabber') - DBOBJ.addHandler(handler) - DBOBJ.setLevel(level) - except (KeyError, ImportError, ValueError): - DBOBJ = None - set_logger(DBOBJ) - -_init_default_logger() -######################################################################## -# END MODULE INITIALIZATION -######################################################################## - - - -class URLGrabError(IOError): - """ - URLGrabError error codes: - - URLGrabber error codes (0 -- 255) - 0 - everything looks good (you should never see this) - 1 - malformed url - 2 - local file doesn't exist - 3 - request for non-file local file (dir, etc) - 4 - IOError on fetch - 5 - OSError on fetch - 6 - no content length header when we expected one - 7 - HTTPException - 8 - Exceeded read limit (for urlread) - 9 - Requested byte range not satisfiable. - 10 - Byte range requested, but range support unavailable - 11 - Illegal reget mode - 12 - Socket timeout - 13 - malformed proxy url - 14 - HTTPError (includes .code and .exception attributes) - 15 - user abort - - MirrorGroup error codes (256 -- 511) - 256 - No more mirrors left to try - - Custom (non-builtin) classes derived from MirrorGroup (512 -- 767) - [ this range reserved for application-specific error codes ] - - Retry codes (< 0) - -1 - retry the download, unknown reason - - Note: to test which group a code is in, you can simply do integer - division by 256: e.errno / 256 - - Negative codes are reserved for use by functions passed in to - retrygrab with checkfunc. The value -1 is built in as a generic - retry code and is already included in the retrycodes list. - Therefore, you can create a custom check function that simply - returns -1 and the fetch will be re-tried. For more customized - retries, you can use other negative number and include them in - retry-codes. This is nice for outputting useful messages about - what failed. - - You can use these error codes like so: - try: urlgrab(url) - except URLGrabError, e: - if e.errno == 3: ... - # or - print e.strerror - # or simply - print e #### print '[Errno %i] %s' % (e.errno, e.strerror) - """ - pass - -class CallbackObject: - """Container for returned callback data. - - This is currently a dummy class into which urlgrabber can stuff - information for passing to callbacks. This way, the prototype for - all callbacks is the same, regardless of the data that will be - passed back. Any function that accepts a callback function as an - argument SHOULD document what it will define in this object. - - It is possible that this class will have some greater - functionality in the future. - """ - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - -def urlgrab(url, filename=None, **kwargs): - """grab the file at and make a local copy at - If filename is none, the basename of the url is used. - urlgrab returns the filename of the local file, which may be different - from the passed-in filename if the copy_local kwarg == 0. - - See module documentation for a description of possible kwargs. - """ - return default_grabber.urlgrab(url, filename, **kwargs) - -def urlopen(url, **kwargs): - """open the url and return a file object - If a progress object or throttle specifications exist, then - a special file object will be returned that supports them. - The file object can be treated like any other file object. - - See module documentation for a description of possible kwargs. - """ - return default_grabber.urlopen(url, **kwargs) - -def urlread(url, limit=None, **kwargs): - """read the url into a string, up to 'limit' bytes - If the limit is exceeded, an exception will be thrown. Note that urlread - is NOT intended to be used as a way of saying "I want the first N bytes" - but rather 'read the whole file into memory, but don't use too much' - - See module documentation for a description of possible kwargs. - """ - return default_grabber.urlread(url, limit, **kwargs) - - -class URLParser: - """Process the URLs before passing them to urllib2. - - This class does several things: - - * add any prefix - * translate a "raw" file to a proper file: url - * handle any http or https auth that's encoded within the url - * quote the url - - Only the "parse" method is called directly, and it calls sub-methods. - - An instance of this class is held in the options object, which - means that it's easy to change the behavior by sub-classing and - passing the replacement in. It need only have a method like: - - url, parts = urlparser.parse(url, opts) - """ - - def parse(self, url, opts): - """parse the url and return the (modified) url and its parts - - Note: a raw file WILL be quoted when it's converted to a URL. - However, other urls (ones which come with a proper scheme) may - or may not be quoted according to opts.quote - - opts.quote = 1 --> quote it - opts.quote = 0 --> do not quote it - opts.quote = None --> guess - """ - quote = opts.quote - - if opts.prefix: - url = self.add_prefix(url, opts.prefix) - - parts = urlparse.urlparse(url) - (scheme, host, path, parm, query, frag) = parts - - if not scheme or (len(scheme) == 1 and scheme in string.letters): - # if a scheme isn't specified, we guess that it's "file:" - if url[0] not in '/\\': url = os.path.abspath(url) - url = 'file:' + urllib.pathname2url(url) - parts = urlparse.urlparse(url) - quote = 0 # pathname2url quotes, so we won't do it again - - if scheme in ['http', 'https']: - parts = self.process_http(parts) - - if quote is None: - quote = self.guess_should_quote(parts) - if quote: - parts = self.quote(parts) - - url = urlparse.urlunparse(parts) - return url, parts - - def add_prefix(self, url, prefix): - if prefix[-1] == '/' or url[0] == '/': - url = prefix + url - else: - url = prefix + '/' + url - return url - - def process_http(self, parts): - (scheme, host, path, parm, query, frag) = parts - - if '@' in host and auth_handler: - try: - user_pass, host = host.split('@', 1) - if ':' in user_pass: - user, password = user_pass.split(':', 1) - except ValueError, e: - raise URLGrabError(1, _('Bad URL: %s') % url) - if DEBUG: DEBUG.info('adding HTTP auth: %s, XXXXXXXX', user) - auth_handler.add_password(None, host, user, password) - - return (scheme, host, path, parm, query, frag) - - def quote(self, parts): - """quote the URL - - This method quotes ONLY the path part. If you need to quote - other parts, you should override this and pass in your derived - class. The other alternative is to quote other parts before - passing into urlgrabber. - """ - (scheme, host, path, parm, query, frag) = parts - path = urllib.quote(path) - return (scheme, host, path, parm, query, frag) - - hexvals = '0123456789ABCDEF' - def guess_should_quote(self, parts): - """ - Guess whether we should quote a path. This amounts to - guessing whether it's already quoted. - - find ' ' -> 1 - find '%' -> 1 - find '%XX' -> 0 - else -> 1 - """ - (scheme, host, path, parm, query, frag) = parts - if ' ' in path: - return 1 - ind = string.find(path, '%') - if ind > -1: - while ind > -1: - if len(path) < ind+3: - return 1 - code = path[ind+1:ind+3].upper() - if code[0] not in self.hexvals or \ - code[1] not in self.hexvals: - return 1 - ind = string.find(path, '%', ind+1) - return 0 - return 1 - -class URLGrabberOptions: - """Class to ease kwargs handling.""" - - def __init__(self, delegate=None, **kwargs): - """Initialize URLGrabberOptions object. - Set default values for all options and then update options specified - in kwargs. - """ - self.delegate = delegate - if delegate is None: - self._set_defaults() - self._set_attributes(**kwargs) - - def __getattr__(self, name): - if self.delegate and hasattr(self.delegate, name): - return getattr(self.delegate, name) - raise AttributeError, name - - def raw_throttle(self): - """Calculate raw throttle value from throttle and bandwidth - values. - """ - if self.throttle <= 0: - return 0 - elif type(self.throttle) == type(0): - return float(self.throttle) - else: # throttle is a float - return self.bandwidth * self.throttle - - def derive(self, **kwargs): - """Create a derived URLGrabberOptions instance. - This method creates a new instance and overrides the - options specified in kwargs. - """ - return URLGrabberOptions(delegate=self, **kwargs) - - def _set_attributes(self, **kwargs): - """Update object attributes with those provided in kwargs.""" - self.__dict__.update(kwargs) - if have_range and kwargs.has_key('range'): - # normalize the supplied range value - self.range = range_tuple_normalize(self.range) - if not self.reget in [None, 'simple', 'check_timestamp']: - raise URLGrabError(11, _('Illegal reget mode: %s') \ - % (self.reget, )) - - def _set_defaults(self): - """Set all options to their default values. - When adding new options, make sure a default is - provided here. - """ - self.progress_obj = None - self.throttle = 1.0 - self.bandwidth = 0 - self.retry = None - self.retrycodes = [-1,2,4,5,6,7] - self.checkfunc = None - self.copy_local = 0 - self.close_connection = 0 - self.range = None - self.user_agent = 'urlgrabber/%s' % __version__ - self.keepalive = 1 - self.proxies = None - self.reget = None - self.failure_callback = None - self.interrupt_callback = None - self.prefix = None - self.opener = None - self.cache_openers = True - self.timeout = None - self.text = None - self.http_headers = None - self.ftp_headers = None - self.data = None - self.urlparser = URLParser() - self.quote = None - self.ssl_ca_cert = None - self.ssl_context = None - -class URLGrabber: - """Provides easy opening of URLs with a variety of options. - - All options are specified as kwargs. Options may be specified when - the class is created and may be overridden on a per request basis. - - New objects inherit default values from default_grabber. - """ - - def __init__(self, **kwargs): - self.opts = URLGrabberOptions(**kwargs) - - def _retry(self, opts, func, *args): - tries = 0 - while 1: - # there are only two ways out of this loop. The second has - # several "sub-ways" - # 1) via the return in the "try" block - # 2) by some exception being raised - # a) an excepton is raised that we don't "except" - # b) a callback raises ANY exception - # c) we're not retry-ing or have run out of retries - # d) the URLGrabError code is not in retrycodes - # beware of infinite loops :) - tries = tries + 1 - exception = None - retrycode = None - callback = None - if DEBUG: DEBUG.info('attempt %i/%s: %s', - tries, opts.retry, args[0]) - try: - r = apply(func, (opts,) + args, {}) - if DEBUG: DEBUG.info('success') - return r - except URLGrabError, e: - exception = e - callback = opts.failure_callback - retrycode = e.errno - except KeyboardInterrupt, e: - exception = e - callback = opts.interrupt_callback - - if DEBUG: DEBUG.info('exception: %s', exception) - if callback: - if DEBUG: DEBUG.info('calling callback: %s', callback) - cb_func, cb_args, cb_kwargs = self._make_callback(callback) - obj = CallbackObject(exception=exception, url=args[0], - tries=tries, retry=opts.retry) - cb_func(obj, *cb_args, **cb_kwargs) - - if (opts.retry is None) or (tries == opts.retry): - if DEBUG: DEBUG.info('retries exceeded, re-raising') - raise - - if (retrycode is not None) and (retrycode not in opts.retrycodes): - if DEBUG: DEBUG.info('retrycode (%i) not in list %s, re-raising', - retrycode, opts.retrycodes) - raise - - def urlopen(self, url, **kwargs): - """open the url and return a file object - If a progress object or throttle value specified when this - object was created, then a special file object will be - returned that supports them. The file object can be treated - like any other file object. - """ - opts = self.opts.derive(**kwargs) - (url,parts) = opts.urlparser.parse(url, opts) - def retryfunc(opts, url): - return URLGrabberFileObject(url, filename=None, opts=opts) - return self._retry(opts, retryfunc, url) - - def urlgrab(self, url, filename=None, **kwargs): - """grab the file at and make a local copy at - If filename is none, the basename of the url is used. - urlgrab returns the filename of the local file, which may be - different from the passed-in filename if copy_local == 0. - """ - opts = self.opts.derive(**kwargs) - (url,parts) = opts.urlparser.parse(url, opts) - (scheme, host, path, parm, query, frag) = parts - if filename is None: - filename = os.path.basename( urllib.unquote(path) ) - if scheme == 'file' and not opts.copy_local: - # just return the name of the local file - don't make a - # copy currently - path = urllib.url2pathname(path) - if host: - path = os.path.normpath('//' + host + path) - if not os.path.exists(path): - raise URLGrabError(2, - _('Local file does not exist: %s') % (path, )) - elif not os.path.isfile(path): - raise URLGrabError(3, - _('Not a normal file: %s') % (path, )) - elif not opts.range: - return path - - def retryfunc(opts, url, filename): - fo = URLGrabberFileObject(url, filename, opts) - try: - fo._do_grab() - if not opts.checkfunc is None: - cb_func, cb_args, cb_kwargs = \ - self._make_callback(opts.checkfunc) - obj = CallbackObject() - obj.filename = filename - obj.url = url - apply(cb_func, (obj, )+cb_args, cb_kwargs) - finally: - fo.close() - return filename - - return self._retry(opts, retryfunc, url, filename) - - def urlread(self, url, limit=None, **kwargs): - """read the url into a string, up to 'limit' bytes - If the limit is exceeded, an exception will be thrown. Note - that urlread is NOT intended to be used as a way of saying - "I want the first N bytes" but rather 'read the whole file - into memory, but don't use too much' - """ - opts = self.opts.derive(**kwargs) - (url,parts) = opts.urlparser.parse(url, opts) - if limit is not None: - limit = limit + 1 - - def retryfunc(opts, url, limit): - fo = URLGrabberFileObject(url, filename=None, opts=opts) - s = '' - try: - # this is an unfortunate thing. Some file-like objects - # have a default "limit" of None, while the built-in (real) - # file objects have -1. They each break the other, so for - # now, we just force the default if necessary. - if limit is None: s = fo.read() - else: s = fo.read(limit) - - if not opts.checkfunc is None: - cb_func, cb_args, cb_kwargs = \ - self._make_callback(opts.checkfunc) - obj = CallbackObject() - obj.data = s - obj.url = url - apply(cb_func, (obj, )+cb_args, cb_kwargs) - finally: - fo.close() - return s - - s = self._retry(opts, retryfunc, url, limit) - if limit and len(s) > limit: - raise URLGrabError(8, - _('Exceeded limit (%i): %s') % (limit, url)) - return s - - def _make_callback(self, callback_obj): - if callable(callback_obj): - return callback_obj, (), {} - else: - return callback_obj - -# create the default URLGrabber used by urlXXX functions. -# NOTE: actual defaults are set in URLGrabberOptions -default_grabber = URLGrabber() - -class URLGrabberFileObject: - """This is a file-object wrapper that supports progress objects - and throttling. - - This exists to solve the following problem: lets say you want to - drop-in replace a normal open with urlopen. You want to use a - progress meter and/or throttling, but how do you do that without - rewriting your code? Answer: urlopen will return a wrapped file - object that does the progress meter and-or throttling internally. - """ - - def __init__(self, url, filename, opts): - self.url = url - self.filename = filename - self.opts = opts - self.fo = None - self._rbuf = '' - self._rbufsize = 1024*8 - self._ttime = time.time() - self._tsize = 0 - self._amount_read = 0 - self._opener = None - self._do_open() - - def __getattr__(self, name): - """This effectively allows us to wrap at the instance level. - Any attribute not found in _this_ object will be searched for - in self.fo. This includes methods.""" - if hasattr(self.fo, name): - return getattr(self.fo, name) - raise AttributeError, name - - def _get_opener(self): - """Build a urllib2 OpenerDirector based on request options.""" - if self.opts.opener: - return self.opts.opener - elif self._opener is None: - handlers = [] - need_keepalive_handler = (have_keepalive and self.opts.keepalive) - need_range_handler = (range_handlers and \ - (self.opts.range or self.opts.reget)) - # if you specify a ProxyHandler when creating the opener - # it _must_ come before all other handlers in the list or urllib2 - # chokes. - if self.opts.proxies: - handlers.append( CachedProxyHandler(self.opts.proxies) ) - - # ------------------------------------------------------- - # OK, these next few lines are a serious kludge to get - # around what I think is a bug in python 2.2's - # urllib2. The basic idea is that default handlers - # get applied first. If you override one (like a - # proxy handler), then the default gets pulled, but - # the replacement goes on the end. In the case of - # proxies, this means the normal handler picks it up - # first and the proxy isn't used. Now, this probably - # only happened with ftp or non-keepalive http, so not - # many folks saw it. The simple approach to fixing it - # is just to make sure you override the other - # conflicting defaults as well. I would LOVE to see - # these go way or be dealt with more elegantly. The - # problem isn't there after 2.2. -MDS 2005/02/24 - if not need_keepalive_handler: - handlers.append( urllib2.HTTPHandler() ) - if not need_range_handler: - handlers.append( urllib2.FTPHandler() ) - # ------------------------------------------------------- - - ssl_factory = sslfactory.get_factory(self.opts.ssl_ca_cert, - self.opts.ssl_context) - - if need_keepalive_handler: - handlers.append(HTTPHandler()) - handlers.append(HTTPSHandler(ssl_factory)) - if need_range_handler: - handlers.extend( range_handlers ) - handlers.append( auth_handler ) - if self.opts.cache_openers: - self._opener = CachedOpenerDirector(ssl_factory, *handlers) - else: - self._opener = ssl_factory.create_opener(*handlers) - # OK, I don't like to do this, but otherwise, we end up with - # TWO user-agent headers. - self._opener.addheaders = [] - return self._opener - - def _do_open(self): - opener = self._get_opener() - - req = urllib2.Request(self.url, self.opts.data) # build request object - self._add_headers(req) # add misc headers that we need - self._build_range(req) #