From f3654023d7b6eb5e522e244f0e1f9d2e1febdf44 Mon Sep 17 00:00:00 2001 From: brice rebsamen Date: Thu, 12 Dec 2013 10:22:17 -0800 Subject: [PATCH 1/7] Update download_checkmd5.py Created a new urlretrieve function that handles retrying if the connection drops. Fixes #559. --- cmake/test/download_checkmd5.py | 124 ++++++++++++++++++++++++++++++-- 1 file changed, 120 insertions(+), 4 deletions(-) diff --git a/cmake/test/download_checkmd5.py b/cmake/test/download_checkmd5.py index 5d0d8160f..936b7a577 100755 --- a/cmake/test/download_checkmd5.py +++ b/cmake/test/download_checkmd5.py @@ -3,16 +3,132 @@ from __future__ import print_function import os import sys -try: - from urllib.request import urlretrieve -except ImportError: - from urllib import urlretrieve +import urllib2 +import time import hashlib from optparse import OptionParser NAME = "download_checkmd5.py" +# Support retry on dropped connection: #559 +class HTTPRangeHandler(urllib2.BaseHandler): + """ + handler that enables HTTP Range headers. + """ + # Taken from urlgrabber: http://urlgrabber.baseurl.org + + def http_error_206(self, req, fp, code, msg, hdrs): + # 206 Partial Content Response + r = urllib.addinfourl(fp, hdrs, req.get_full_url()) + r.code = code + r.msg = msg + return r + + def http_error_416(self, req, fp, code, msg, hdrs): + # HTTP's Range Not Satisfiable error + raise RangeError('Requested Range Not Satisfiable') + + +def urlretrieve(url, localfile=None, nRetry=20, append=False, progressFn=None): + """ + downloads the file pointed to by the url. Supports retrying and resuming + a partially downloaded file. + + Arguments: + ---------- + + url: string or anything supported by urllib + The URL of the file to download. + localfile: string (defaults to None) + The name of the resulting file. If None, the basename of the url + is used. + nRetry: integer (defaults to 20) + The max number of trials. + append: boolean (defaults to False) + In case the localfile exists, whether we want to append to it, + i.e. resume a partial download. If false, then the localfile is + first deleted. + progressFn: function (defaults to None) + If not None, the function is called when new data is downloaded, + with 2 arguments: the number of bytes downloaded so far and + the total number of bytes. + """ + + # TODO: + # - there is probably a number of errors that can occur when reading + # where we would want to retry. At the moment we just fail. + # - If the range header is not supported, an exception is thrown. + # There might be a better strategy. For instance: delete the localfile + # and attempt to download it again from scratch ... + + if localfile is None: + localfile = url.split('/')[-1] + + # If the file exists but we do not wish to append to it then erase it. + if not append and os.path.exists(localfile) and os.path.isfile(localfile): + os.remove(localfile) + + range_handler = HTTPRangeHandler() + opener = urllib2.build_opener(range_handler) + + for retry in range(nRetry): + _netfile = opener.open(url) + filesize = float(_netfile.info()['Content-Length']) + + count = 0 + if os.path.exists(localfile) and os.path.isfile(localfile): + count = os.path.getsize(localfile) + if count >= filesize: + # we are done + _netfile.close() + return + + #print("resuming: %d of %d downloaded." % (count, filesize)) + _netfile.close() + req = urllib2.Request(url) + req.add_header("Range","bytes=%s-" % (count)) + _netfile = opener.open(req) + + _outfile = open(localfile, "ab") + data = None + chunk_size = 8 * 1024 + while data is None or len(data)>0: + data = _netfile.read(chunk_size) + _outfile.write(data) + count += len(data) + + if progressFn: + progressFn(count, filesize) + + if count >= filesize: + # we are done + _netfile.close() + _outfile.close() + return + + # we reach this point if the read returned no bytes. In which case we + # want to retry + _netfile.close() + _outfile.close() + + # is it necessary ??? + time.sleep(1) + + raise IOError('Failed to download file. Exceeded number of trials.') + + +def progress_report(bytes_so_far, total_size): + """ + progress report function that can be used with urlretrieve(). + """ + percent = float(bytes_so_far) / total_size + percent = round(percent*100, 2) + sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" % (bytes_so_far, total_size, percent)) + if bytes_so_far >= total_size: + sys.stdout.write('\n') + + def download_md5(uri, dest): """ downloads file from uri to file dest From 50d4afbc8babd5183eb063b95e309a4dd08d098e Mon Sep 17 00:00:00 2001 From: brice rebsamen Date: Thu, 12 Dec 2013 11:35:10 -0800 Subject: [PATCH 2/7] Update download_checkmd5.py python3 compatibility (tested) --- cmake/test/download_checkmd5.py | 62 +++++++++++++++++---------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/cmake/test/download_checkmd5.py b/cmake/test/download_checkmd5.py index 936b7a577..ad5ea0782 100755 --- a/cmake/test/download_checkmd5.py +++ b/cmake/test/download_checkmd5.py @@ -3,7 +3,10 @@ from __future__ import print_function import os import sys -import urllib2 +try: + from urllib2 import urlopen, BaseHandler, build_opener, Request +except ImportError: + from urllib.request import urlopen, BaseHandler, build_opener, Request import time import hashlib from optparse import OptionParser @@ -12,7 +15,7 @@ # Support retry on dropped connection: #559 -class HTTPRangeHandler(urllib2.BaseHandler): +class HTTPRangeHandler(BaseHandler): """ handler that enables HTTP Range headers. """ @@ -28,49 +31,48 @@ def http_error_206(self, req, fp, code, msg, hdrs): def http_error_416(self, req, fp, code, msg, hdrs): # HTTP's Range Not Satisfiable error raise RangeError('Requested Range Not Satisfiable') - -def urlretrieve(url, localfile=None, nRetry=20, append=False, progressFn=None): - """ + +def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None): + """ downloads the file pointed to by the url. Supports retrying and resuming a partially downloaded file. Arguments: ---------- - url: string or anything supported by urllib - The URL of the file to download. + The URL of the file to download. localfile: string (defaults to None) - The name of the resulting file. If None, the basename of the url - is used. + The name of the resulting file. If None, the basename of the url + is used. nRetry: integer (defaults to 20) - The max number of trials. + The max number of trials. append: boolean (defaults to False) - In case the localfile exists, whether we want to append to it, - i.e. resume a partial download. If false, then the localfile is - first deleted. + In case the localfile exists, whether we want to append to it, + i.e. resume a partial download. If false, then the localfile is + first deleted. progressFn: function (defaults to None) - If not None, the function is called when new data is downloaded, - with 2 arguments: the number of bytes downloaded so far and - the total number of bytes. + If not None, the function is called when new data is downloaded, + with 2 arguments: the number of bytes downloaded so far and + the total number of bytes. """ # TODO: - # - there is probably a number of errors that can occur when reading - # where we would want to retry. At the moment we just fail. - # - If the range header is not supported, an exception is thrown. - # There might be a better strategy. For instance: delete the localfile - # and attempt to download it again from scratch ... + # - there is probably a number of errors that can occur when reading + # where we would want to retry. At the moment we just fail. + # - If the range header is not supported, an exception is thrown. + # There might be a better strategy. For instance: delete the localfile + # and attempt to download it again from scratch ... if localfile is None: localfile = url.split('/')[-1] - + # If the file exists but we do not wish to append to it then erase it. if not append and os.path.exists(localfile) and os.path.isfile(localfile): os.remove(localfile) range_handler = HTTPRangeHandler() - opener = urllib2.build_opener(range_handler) + opener = build_opener(range_handler) for retry in range(nRetry): _netfile = opener.open(url) @@ -83,10 +85,10 @@ def urlretrieve(url, localfile=None, nRetry=20, append=False, progressFn=None): # we are done _netfile.close() return - + #print("resuming: %d of %d downloaded." % (count, filesize)) _netfile.close() - req = urllib2.Request(url) + req = Request(url) req.add_header("Range","bytes=%s-" % (count)) _netfile = opener.open(req) @@ -114,13 +116,13 @@ def urlretrieve(url, localfile=None, nRetry=20, append=False, progressFn=None): # is it necessary ??? time.sleep(1) - + raise IOError('Failed to download file. Exceeded number of trials.') def progress_report(bytes_so_far, total_size): - """ - progress report function that can be used with urlretrieve(). + """ + progress report function that can be used with download_file(). """ percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) @@ -140,7 +142,7 @@ def download_md5(uri, dest): sys.stdout.write('Downloading %s to %s...' % (uri, dest)) sys.stdout.flush() - urlretrieve(uri, dest) + download_file(uri, dest) sys.stdout.write('Done\n') @@ -190,7 +192,7 @@ def main(argv=sys.argv[1:]): download_md5(uri, dest) result, hexdigest = checkmd5(dest, md5sum) if result is False: - sys.exit('ERROR: md5sum mismatch (%s != %s) on %s; aborting' % (hexdigest, md5sum, dest)) + sys.exit('ERROR: md5sum mismatch (%s != %s) on %s; aborting' % (hexdigest, md5sum, dest)) if __name__ == '__main__': From e2829af50e6adf9e3f2dc65779b6f7166638fb28 Mon Sep 17 00:00:00 2001 From: brice rebsamen Date: Thu, 12 Dec 2013 16:27:59 -0800 Subject: [PATCH 3/7] Update download_checkmd5.py fixed some more problems pointed by dirk --- cmake/test/download_checkmd5.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/cmake/test/download_checkmd5.py b/cmake/test/download_checkmd5.py index ad5ea0782..460711ceb 100755 --- a/cmake/test/download_checkmd5.py +++ b/cmake/test/download_checkmd5.py @@ -4,9 +4,9 @@ import os import sys try: - from urllib2 import urlopen, BaseHandler, build_opener, Request + from urllib2 import urlopen, BaseHandler, build_opener, Request, addinfourl except ImportError: - from urllib.request import urlopen, BaseHandler, build_opener, Request + from urllib.request import urlopen, BaseHandler, build_opener, Request, addinfourl import time import hashlib from optparse import OptionParser @@ -23,7 +23,7 @@ class HTTPRangeHandler(BaseHandler): def http_error_206(self, req, fp, code, msg, hdrs): # 206 Partial Content Response - r = urllib.addinfourl(fp, hdrs, req.get_full_url()) + r = addinfourl(fp, hdrs, req.get_full_url()) r.code = code r.msg = msg return r @@ -71,6 +71,8 @@ def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None) if not append and os.path.exists(localfile) and os.path.isfile(localfile): os.remove(localfile) + lastProgressReportPercent = None + range_handler = HTTPRangeHandler() opener = build_opener(range_handler) @@ -81,10 +83,12 @@ def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None) count = 0 if os.path.exists(localfile) and os.path.isfile(localfile): count = os.path.getsize(localfile) - if count >= filesize: + if count == filesize: # we are done _netfile.close() return + elif count > filesize: + raise IOError('Resulting file is larger than source file') #print("resuming: %d of %d downloaded." % (count, filesize)) _netfile.close() @@ -100,10 +104,13 @@ def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None) _outfile.write(data) count += len(data) - if progressFn: - progressFn(count, filesize) + if progressFn is not None: + percent = float(count)/filesize*100 + if lastProgressReportPercent is None or percent >= lastProgressReportPercent+1: + progressFn(count, filesize) + lastProgressReportPercent = int(float(count)/filesize*100) - if count >= filesize: + if count == filesize: # we are done _netfile.close() _outfile.close() @@ -114,7 +121,9 @@ def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None) _netfile.close() _outfile.close() - # is it necessary ??? + # wget waits 1 second after the first failure, 2 seconds after the + # the second, etc. + # here we will just wait 1 second each time. time.sleep(1) raise IOError('Failed to download file. Exceeded number of trials.') @@ -127,6 +136,7 @@ def progress_report(bytes_so_far, total_size): percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" % (bytes_so_far, total_size, percent)) + sys.stdout.flush() if bytes_so_far >= total_size: sys.stdout.write('\n') From 27026d4dd1da08f56be219815c0ff1695c68a7b7 Mon Sep 17 00:00:00 2001 From: Brice Rebsamen Date: Wed, 25 Dec 2013 14:12:28 -0800 Subject: [PATCH 4/7] Fixed problem with local file URL --- cmake/test/download_checkmd5.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmake/test/download_checkmd5.py b/cmake/test/download_checkmd5.py index 460711ceb..41868936c 100755 --- a/cmake/test/download_checkmd5.py +++ b/cmake/test/download_checkmd5.py @@ -64,6 +64,13 @@ def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None) # There might be a better strategy. For instance: delete the localfile # and attempt to download it again from scratch ... + + # The 'file://' prefix is mandatory with the our HTTPRangeHandler + # (used to be optional with urllib.urlretrieve). + # Restoring for backward compatibility. + if '://' not in url: + url = 'file://' + url + if localfile is None: localfile = url.split('/')[-1] From 06f69be5e3db7cf3874f755a1038e9145dc31c4e Mon Sep 17 00:00:00 2001 From: Brice Rebsamen Date: Wed, 25 Dec 2013 20:47:33 -0800 Subject: [PATCH 5/7] Changed the retrial policy: count the number of consecutive failures to download. --- cmake/test/download_checkmd5.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/cmake/test/download_checkmd5.py b/cmake/test/download_checkmd5.py index 41868936c..d7b95d6e2 100755 --- a/cmake/test/download_checkmd5.py +++ b/cmake/test/download_checkmd5.py @@ -33,25 +33,25 @@ def http_error_416(self, req, fp, code, msg, hdrs): raise RangeError('Requested Range Not Satisfiable') -def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None): +def download_file(url, localfile=None, nRetry=3, append=False, progressFn=None): """ downloads the file pointed to by the url. Supports retrying and resuming a partially downloaded file. Arguments: ---------- - url: string or anything supported by urllib + - url: string or anything supported by urllib The URL of the file to download. - localfile: string (defaults to None) + - localfile: string (defaults to None) The name of the resulting file. If None, the basename of the url is used. - nRetry: integer (defaults to 20) - The max number of trials. - append: boolean (defaults to False) + - nRetry: integer (defaults to 3) + The max number of consecutive failures to download some bytes from the url. + - append: boolean (defaults to False) In case the localfile exists, whether we want to append to it, i.e. resume a partial download. If false, then the localfile is first deleted. - progressFn: function (defaults to None) + - progressFn: function (defaults to None) If not None, the function is called when new data is downloaded, with 2 arguments: the number of bytes downloaded so far and the total number of bytes. @@ -83,7 +83,8 @@ def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None) range_handler = HTTPRangeHandler() opener = build_opener(range_handler) - for retry in range(nRetry): + retry = 0 + while retry < nRetry: _netfile = opener.open(url) filesize = float(_netfile.info()['Content-Length']) @@ -97,7 +98,7 @@ def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None) elif count > filesize: raise IOError('Resulting file is larger than source file') - #print("resuming: %d of %d downloaded." % (count, filesize)) + #print("\n\nresuming: %d of %d downloaded.\n" % (count, filesize)) _netfile.close() req = Request(url) req.add_header("Range","bytes=%s-" % (count)) @@ -108,6 +109,7 @@ def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None) chunk_size = 8 * 1024 while data is None or len(data)>0: data = _netfile.read(chunk_size) + if len(data)>0: retry = 0 _outfile.write(data) count += len(data) @@ -127,6 +129,7 @@ def download_file(url, localfile=None, nRetry=20, append=False, progressFn=None) # want to retry _netfile.close() _outfile.close() + retry = retry + 1 # wget waits 1 second after the first failure, 2 seconds after the # the second, etc. @@ -160,6 +163,7 @@ def download_md5(uri, dest): sys.stdout.write('Downloading %s to %s...' % (uri, dest)) sys.stdout.flush() download_file(uri, dest) + #download_file(uri, dest, progressFn=progress_report) sys.stdout.write('Done\n') From 15a872c584a715aef887353c5c441df00c4a42d8 Mon Sep 17 00:00:00 2001 From: Brice Rebsamen Date: Thu, 26 Dec 2013 02:08:00 -0800 Subject: [PATCH 6/7] Changed to an object oriented code, and checking whether the server supports partial downloads. --- cmake/test/download_checkmd5.py | 266 ++++++++++++++++++++------------ 1 file changed, 166 insertions(+), 100 deletions(-) diff --git a/cmake/test/download_checkmd5.py b/cmake/test/download_checkmd5.py index d7b95d6e2..018b22d58 100755 --- a/cmake/test/download_checkmd5.py +++ b/cmake/test/download_checkmd5.py @@ -4,9 +4,10 @@ import os import sys try: - from urllib2 import urlopen, BaseHandler, build_opener, Request, addinfourl + from urllib.request import urlopen, BaseHandler, build_opener, Request, addinfourl, urlretrieve except ImportError: - from urllib.request import urlopen, BaseHandler, build_opener, Request, addinfourl + from urllib2 import urlopen, BaseHandler, build_opener, Request, addinfourl + from urllib import urlretrieve import time import hashlib from optparse import OptionParser @@ -33,110 +34,172 @@ def http_error_416(self, req, fp, code, msg, hdrs): raise RangeError('Requested Range Not Satisfiable') -def download_file(url, localfile=None, nRetry=3, append=False, progressFn=None): +class FileDownloader: """ - downloads the file pointed to by the url. Supports retrying and resuming - a partially downloaded file. - - Arguments: - ---------- - - url: string or anything supported by urllib - The URL of the file to download. - - localfile: string (defaults to None) - The name of the resulting file. If None, the basename of the url - is used. + File downloader. + + Parameters: + ----------- - nRetry: integer (defaults to 3) - The max number of consecutive failures to download some bytes from the url. + The max number of consecutive failures to download some bytes from the url. - append: boolean (defaults to False) - In case the localfile exists, whether we want to append to it, - i.e. resume a partial download. If false, then the localfile is - first deleted. + In case the localfile exists, whether we want to append to it, + i.e. resume a partial download. If false, then the localfile is + first deleted. - progressFn: function (defaults to None) - If not None, the function is called when new data is downloaded, - with 2 arguments: the number of bytes downloaded so far and - the total number of bytes. + If not None, the function is called when new data is downloaded, + with 2 arguments: the number of bytes downloaded so far and + the total number of bytes. """ - - # TODO: - # - there is probably a number of errors that can occur when reading - # where we would want to retry. At the moment we just fail. - # - If the range header is not supported, an exception is thrown. - # There might be a better strategy. For instance: delete the localfile - # and attempt to download it again from scratch ... - - # The 'file://' prefix is mandatory with the our HTTPRangeHandler - # (used to be optional with urllib.urlretrieve). - # Restoring for backward compatibility. - if '://' not in url: - url = 'file://' + url - - if localfile is None: - localfile = url.split('/')[-1] - - # If the file exists but we do not wish to append to it then erase it. - if not append and os.path.exists(localfile) and os.path.isfile(localfile): - os.remove(localfile) + def __init__(self): + """ + Constructor: provides default value for the parameters. + """ + self.nRetry = 3 + self.append = False + self.progressFn = None + self.chunk_size = 8 * 1024 + self.verbose = False + + + def download_file(self, url, localfile=None): + """ + downloads the file pointed to by the url. Supports retrying and resuming + a partially downloaded file. + + Arguments: + ---------- + - url: string or anything supported by urllib + The URL of the file to download. + - localfile: string (defaults to None) + The name of the resulting file. If None, the basename of the url + is used. + """ + + self.count = 0 + self.lastProgressReportPercent = None + self.retry = 0 + + self.url = url + # The 'file://' prefix is mandatory with the our HTTPRangeHandler + # (used to be optional with urllib.urlretrieve). + # Restoring for backward compatibility. + if '://' not in self.url: + self.url = 'file://' + self.url + + self.localfile = localfile + if self.localfile is None: + self.localfile = os.path.basename(self.url) + + # If the file exists but we do not wish to append to it then erase it. + # If we want to append to it, then get its size. + if os.path.exists(self.localfile) and os.path.isfile(self.localfile): + if self.append: + self.count = os.path.getsize(self.localfile) + else: + os.remove(self.localfile) + + self.range_handler = HTTPRangeHandler() + self.opener = build_opener(self.range_handler) + + # Check if the server provides the remote file size and if it supports + # range header. If it does not, then fallback to using urlretrieve + # (and download the whole file without being able to recover from errors) + netfile = self._connect(self.url) + header = netfile.info() + netfile.close() + if not header.has_key('Content-Length') or not header.has_key('Accept-Ranges') or header['Accept-Ranges']=='none': + urlretrieve(self.url, self.localfile) + return + + self.filesize = int(header['Content-Length']) + if self.count == self.filesize: + return + elif self.count > self.filesize: + raise IOError('Resulting file is larger than source file') + + while not self._download(): pass + + + def _connect(self, req): + """ + Tries to open a connection to the server (with retrial). + Returns the file-like object returned by urlopen on success. Raises + an IOError if the number of retrials has been exceeded. + """ + while True: + try: + return self.opener.open(req) + except: + # As wget does, we wait 1 second after the 1st failure, 2 seconds + # after the 2nd, etc. + self.retry = self.retry + 1 + if self.retry < self.nRetry: + time.sleep(retry) + else: + raise IOError('Failed to download file. Exceeded number of trials.') + + + def _progress(self): + if self.progressFn is not None: + percent = float(self.count)/self.filesize*100 + if self.lastProgressReportPercent is None or percent >= self.lastProgressReportPercent+1: + self.progressFn(self.count, self.filesize) + self.lastProgressReportPercent = int(float(self.count)/self.filesize*100) + + + def _check_content_range(self, header): + """ + Checks that the response from the server contains a valid Content-Range + field. Raises an IOError if it does not. + """ + msg = None + if not header.has_key('Content-Range'): + msg = 'server did not reply with Content-Range' + else: + try: + if int(header['Content-Range'][len('bytes '):].split('-')[0]) != self.count: + msg = 'server replied with a Content-Range starting at the wrong place' + except: + msg = 'failed to parse the Content-Range header' + if msg: + raise IOError('Failed to resume the download: %s.' % msg) - lastProgressReportPercent = None - range_handler = HTTPRangeHandler() - opener = build_opener(range_handler) - - retry = 0 - while retry < nRetry: - _netfile = opener.open(url) - filesize = float(_netfile.info()['Content-Length']) - - count = 0 - if os.path.exists(localfile) and os.path.isfile(localfile): - count = os.path.getsize(localfile) - if count == filesize: - # we are done - _netfile.close() - return - elif count > filesize: - raise IOError('Resulting file is larger than source file') - - #print("\n\nresuming: %d of %d downloaded.\n" % (count, filesize)) - _netfile.close() - req = Request(url) - req.add_header("Range","bytes=%s-" % (count)) - _netfile = opener.open(req) - - _outfile = open(localfile, "ab") - data = None - chunk_size = 8 * 1024 - while data is None or len(data)>0: - data = _netfile.read(chunk_size) - if len(data)>0: retry = 0 - _outfile.write(data) - count += len(data) - - if progressFn is not None: - percent = float(count)/filesize*100 - if lastProgressReportPercent is None or percent >= lastProgressReportPercent+1: - progressFn(count, filesize) - lastProgressReportPercent = int(float(count)/filesize*100) - - if count == filesize: - # we are done - _netfile.close() - _outfile.close() - return - - # we reach this point if the read returned no bytes. In which case we - # want to retry - _netfile.close() - _outfile.close() - retry = retry + 1 - - # wget waits 1 second after the first failure, 2 seconds after the - # the second, etc. - # here we will just wait 1 second each time. - time.sleep(1) - - raise IOError('Failed to download file. Exceeded number of trials.') + def _download(self): + """ + Downloads the next bytes from the file. Returns True if the download + completed, False otherwise in which case we want to reopen the connection + and try again. + Raises IOErrors if something goes wrong. + """ + if self.count>0 and self.verbose: + print("\n\nresuming: %d of %d downloaded.\n" % (self.count, self.filesize)) + + netfile = None + outfile = None + try: + req = Request(self.url) + req.add_header("Range","bytes=%d-" % (self.count)) + netfile = self._connect(req) + self._check_content_range(netfile.info()) + outfile = open(self.localfile, "ab") + data = None + while data is None or len(data)>0: + data = netfile.read(self.chunk_size) + if len(data)>0: self.retry = 0 + outfile.write(data) + self.count += len(data) + self._progress() + if self.count == self.filesize: + return True + elif self.count > self.filesize: + raise IOError('Resulting file is larger than source file') + finally: + if netfile: netfile.close() + if outfile: outfile.close() + return False def progress_report(bytes_so_far, total_size): @@ -162,8 +225,11 @@ def download_md5(uri, dest): sys.stdout.write('Downloading %s to %s...' % (uri, dest)) sys.stdout.flush() - download_file(uri, dest) - #download_file(uri, dest, progressFn=progress_report) + + downloader = FileDownloader() + downloader.progressFn = progress_report + downloader.verbose = True + downloader.download_file(uri, dest) sys.stdout.write('Done\n') From 94bd20c1c0e4713ff224e7da5389b1b2293246a8 Mon Sep 17 00:00:00 2001 From: Brice Rebsamen Date: Thu, 26 Dec 2013 02:11:42 -0800 Subject: [PATCH 7/7] commented out verbosity --- cmake/test/download_checkmd5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/test/download_checkmd5.py b/cmake/test/download_checkmd5.py index 018b22d58..6c3e51f5d 100755 --- a/cmake/test/download_checkmd5.py +++ b/cmake/test/download_checkmd5.py @@ -227,8 +227,8 @@ def download_md5(uri, dest): sys.stdout.flush() downloader = FileDownloader() - downloader.progressFn = progress_report - downloader.verbose = True + #downloader.progressFn = progress_report + #downloader.verbose = True downloader.download_file(uri, dest) sys.stdout.write('Done\n')