From f5884f6824291cda15096d84af9e346e13a41578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyryl=20P=C5=82otnicki-Chudyk?= Date: Sat, 30 Apr 2016 08:09:53 +0200 Subject: [PATCH 1/4] Get a file during bootstrap to a temp location. When downloading a file in the bootstrap phase - get it to a temp location first. Verify it there and only if downloaded properly move it to the `cache` directory. This should prevent `make` being stuck if the download was interrupted or otherwise corrupted. The temporary files are deleted in case of an exception. --- src/bootstrap/bootstrap.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index a05700c1af49..6efaf16a9743 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -16,12 +16,26 @@ import subprocess import sys import tarfile +import tempfile + def get(url, path, verbose=False): - print("downloading " + url) sha_url = url + ".sha256" - sha_path = path + ".sha256" - for _url, _path in ((url, path), (sha_url, sha_path)): + temp_file = tempfile.NamedTemporaryFile(delete=False) + temp_path = temp_file.name + sha_file = tempfile.NamedTemporaryFile(suffix=".sha256", delete=True) + sha_path = sha_file.name + download(sha_path, sha_url, temp_path, url, verbose) + verify(sha_path, temp_path, verbose) + sha_file.close() + print("moving " + temp_path + " to " + path) + shutil.move(temp_path, path) + temp_file.close() + + +def download(sha_path, sha_url, temp_path, url, verbose): + for _url, _path in ((url, temp_path), (sha_url, sha_path)): + print("downloading " + _url + " to " + _path) # see http://serverfault.com/questions/301128/how-to-download if sys.platform == 'win32': run(["PowerShell.exe", "/nologo", "-Command", @@ -30,8 +44,11 @@ def get(url, path, verbose=False): verbose=verbose) else: run(["curl", "-o", _path, _url], verbose=verbose) - print("verifying " + path) - with open(path, "rb") as f: + + +def verify(sha_path, temp_path, verbose): + print("verifying " + temp_path) + with open(temp_path, "rb") as f: found = hashlib.sha256(f.read()).hexdigest() with open(sha_path, "r") as f: expected, _ = f.readline().split() @@ -43,6 +60,7 @@ def get(url, path, verbose=False): raise RuntimeError(err) sys.exit(err) + def unpack(tarball, dst, verbose=False, match=None): print("extracting " + tarball) fname = os.path.basename(tarball).replace(".tar.gz", "") From 18dafe83b5411e76a13f5e813dae1bec71ef07a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyryl=20P=C5=82otnicki-Chudyk?= Date: Sat, 30 Apr 2016 08:43:01 +0200 Subject: [PATCH 2/4] Code cleanup in download() in bootstrap.py Make download() receive less parameters and use it explicitly 2 times in get(). --- src/bootstrap/bootstrap.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 6efaf16a9743..7599afacc827 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -25,7 +25,8 @@ def get(url, path, verbose=False): temp_path = temp_file.name sha_file = tempfile.NamedTemporaryFile(suffix=".sha256", delete=True) sha_path = sha_file.name - download(sha_path, sha_url, temp_path, url, verbose) + download(sha_path, sha_url, verbose) + download(temp_path, url, verbose) verify(sha_path, temp_path, verbose) sha_file.close() print("moving " + temp_path + " to " + path) @@ -33,17 +34,16 @@ def get(url, path, verbose=False): temp_file.close() -def download(sha_path, sha_url, temp_path, url, verbose): - for _url, _path in ((url, temp_path), (sha_url, sha_path)): - print("downloading " + _url + " to " + _path) - # see http://serverfault.com/questions/301128/how-to-download - if sys.platform == 'win32': - run(["PowerShell.exe", "/nologo", "-Command", - "(New-Object System.Net.WebClient)" - ".DownloadFile('{}', '{}')".format(_url, _path)], - verbose=verbose) - else: - run(["curl", "-o", _path, _url], verbose=verbose) +def download(path, url, verbose): + print("downloading " + url + " to " + path) + # see http://serverfault.com/questions/301128/how-to-download + if sys.platform == 'win32': + run(["PowerShell.exe", "/nologo", "-Command", + "(New-Object System.Net.WebClient)" + ".DownloadFile('{}', '{}')".format(url, path)], + verbose=verbose) + else: + run(["curl", "-o", path, url], verbose=verbose) def verify(sha_path, temp_path, verbose): From 15de207b9c79c603aa10c30ae54ca07b295aadaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyryl=20P=C5=82otnicki-Chudyk?= Date: Sun, 8 May 2016 09:54:50 +0200 Subject: [PATCH 3/4] Better error handling for bootstrap file downloads. Remove the temp files if something goes wrong. --- src/bootstrap/bootstrap.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 7599afacc827..c9167d7638a3 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -21,17 +21,23 @@ def get(url, path, verbose=False): sha_url = url + ".sha256" - temp_file = tempfile.NamedTemporaryFile(delete=False) - temp_path = temp_file.name - sha_file = tempfile.NamedTemporaryFile(suffix=".sha256", delete=True) - sha_path = sha_file.name - download(sha_path, sha_url, verbose) - download(temp_path, url, verbose) - verify(sha_path, temp_path, verbose) - sha_file.close() - print("moving " + temp_path + " to " + path) - shutil.move(temp_path, path) - temp_file.close() + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_path = temp_file.name + with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file: + sha_path = sha_file.name + + try: + download(sha_path, sha_url, verbose) + download(temp_path, url, verbose) + verify(temp_path, sha_path, verbose) + print("moving " + temp_path + " to " + path) + shutil.move(temp_path, path) + finally: + print("removing " + sha_path) + os.unlink(sha_path) + if os.path.isfile(temp_path): + print("removing " + temp_path) + os.unlink(temp_path) def download(path, url, verbose): @@ -46,9 +52,9 @@ def download(path, url, verbose): run(["curl", "-o", path, url], verbose=verbose) -def verify(sha_path, temp_path, verbose): - print("verifying " + temp_path) - with open(temp_path, "rb") as f: +def verify(path, sha_path, verbose): + print("verifying " + path) + with open(path, "rb") as f: found = hashlib.sha256(f.read()).hexdigest() with open(sha_path, "r") as f: expected, _ = f.readline().split() From 6bce110c3e6a931fe1bd79c640e9d29a744f0e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyryl=20P=C5=82otnicki-Chudyk?= Date: Sun, 8 May 2016 10:00:36 +0200 Subject: [PATCH 4/4] Remove sha256 file only if present. --- src/bootstrap/bootstrap.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index c9167d7638a3..0ab5253ee723 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -33,11 +33,14 @@ def get(url, path, verbose=False): print("moving " + temp_path + " to " + path) shutil.move(temp_path, path) finally: - print("removing " + sha_path) - os.unlink(sha_path) - if os.path.isfile(temp_path): - print("removing " + temp_path) - os.unlink(temp_path) + delete_if_present(sha_path) + delete_if_present(temp_path) + + +def delete_if_present(path): + if os.path.isfile(path): + print("removing " + path) + os.unlink(path) def download(path, url, verbose):