From e41bf0230e7c5bf58e17692ee2880ff775a164f7 Mon Sep 17 00:00:00 2001
From: Marcus Smith <qwcode@gmail.com>
Date: Sat, 1 Feb 2014 11:41:55 -0800
Subject: [PATCH 1/2] 'pip wheel' should download wheels, when it finds them
---
pip/commands/wheel.py | 11 ++--
pip/download.py | 78 +++++++++++++++++++++++------
pip/req/req_set.py | 84 +++++++++++++++++++++++--------
pip/wheel.py | 14 +++---
tests/functional/test_wheel.py | 17 ++++++-
tests/unit/test_download.py | 91 +++++++++++++++++++++++++++++++++-
6 files changed, 241 insertions(+), 54 deletions(-)
diff --git a/pip/commands/wheel.py b/pip/commands/wheel.py
index edf3ef7ad6b..757891a5b52 100644
--- a/pip/commands/wheel.py
+++ b/pip/commands/wheel.py
@@ -165,13 +165,15 @@ def run(self, options, args):
ignore_dependencies=options.ignore_dependencies,
ignore_installed=True,
session=session,
+ wheel_download_dir=options.wheel_dir
)
+ # make the wheelhouse
+ if not os.path.exists(options.wheel_dir):
+ os.makedirs(options.wheel_dir)
+
#parse args and/or requirements files
for name in args:
- if name.endswith(".whl"):
- logger.notify("ignoring %s" % name)
- continue
requirement_set.add_requirement(
InstallRequirement.from_line(name, None))
@@ -181,8 +183,7 @@ def run(self, options, args):
finder=finder,
options=options,
session=session):
- if (req.editable
- or (req.name is None and req.url.endswith(".whl"))):
+ if req.editable:
logger.notify("ignoring %s" % req.url)
continue
requirement_set.add_requirement(req)
diff --git a/pip/download.py b/pip/download.py
index a7fd71c73df..a9169842acd 100644
--- a/pip/download.py
+++ b/pip/download.py
@@ -348,18 +348,6 @@ def unpack_vcs_link(link, location, only_download=False):
vcs_backend.unpack(location)
-def unpack_file_url(link, location):
- source = url_to_path(link.url)
- content_type = mimetypes.guess_type(source)[0]
- if os.path.isdir(source):
- # delete the location since shutil will create it again :(
- if os.path.isdir(location):
- rmtree(location)
- shutil.copytree(source, location, symlinks=True)
- else:
- unpack_file(source, location, content_type, link)
-
-
def _get_used_vcs_backend(link):
for backend in vcs.backends:
if link.scheme in backend.schemes:
@@ -507,7 +495,6 @@ def _copy_file(filename, location, content_type, link):
shutil.move(download_location, dest_file)
if copy:
shutil.copy(filename, download_location)
- logger.indent -= 2
logger.notify('Saved %s' % display_path(download_location))
@@ -519,11 +506,12 @@ def unpack_http_url(link, location, download_cache, download_dir=None,
temp_dir = tempfile.mkdtemp('-unpack', 'pip-')
temp_location = None
target_url = link.url.split('#', 1)[0]
-
already_cached = False
cache_file = None
cache_content_type_file = None
download_hash = None
+
+ # If a download cache is specified, is the file cached there?
if download_cache:
cache_file = os.path.join(
download_cache,
@@ -537,12 +525,14 @@ def unpack_http_url(link, location, download_cache, download_dir=None,
if not os.path.isdir(download_cache):
create_download_cache_folder(download_cache)
+ # If a download dir is specified, is the file already downloaded there?
already_downloaded = None
if download_dir:
already_downloaded = os.path.join(download_dir, link.filename)
if not os.path.exists(already_downloaded):
already_downloaded = None
+ # If already downloaded, does it's hash match?
if already_downloaded:
temp_location = already_downloaded
content_type = mimetypes.guess_type(already_downloaded)[0]
@@ -560,8 +550,7 @@ def unpack_http_url(link, location, download_cache, download_dir=None,
os.unlink(already_downloaded)
already_downloaded = None
- # We have a cached file, and we haven't already found a good downloaded
- # copy
+ # If not a valid download, let's confirm the cached file is valid
if already_cached and not temp_location:
with open(cache_content_type_file) as fp:
content_type = fp.read().strip()
@@ -582,6 +571,7 @@ def unpack_http_url(link, location, download_cache, download_dir=None,
already_cached = False
# We don't have either a cached or a downloaded copy
+ # let's download to a tmp dir
if not temp_location:
try:
resp = session.get(target_url, stream=True)
@@ -614,11 +604,67 @@ def unpack_http_url(link, location, download_cache, download_dir=None,
if link.hash and link.hash_name:
_check_hash(download_hash, link)
+ # a download dir is specified; let's copy the archive there
if download_dir and not already_downloaded:
_copy_file(temp_location, download_dir, content_type, link)
+
+ # unpack the archive to the build dir location. even when only downloading
+ # archives, they have to be unpacked to parse dependencies
unpack_file(temp_location, location, content_type, link)
+
+ # if using a download cache, cache it, if needed
if cache_file and not already_cached:
cache_download(cache_file, temp_location, content_type)
+
if not (already_cached or already_downloaded):
os.unlink(temp_location)
+
os.rmdir(temp_dir)
+
+
+def unpack_file_url(link, location, download_dir=None):
+
+ link_path = url_to_path(link.url_without_fragment)
+ from_path = None
+ already_downloaded = False
+
+ # If it's a url to a local directory
+ if os.path.isdir(link_path):
+ if os.path.isdir(location):
+ rmtree(location)
+ shutil.copytree(link_path, location, symlinks=True)
+ return
+
+ # If a download dir is specified, is the file already there and valid?
+ if download_dir:
+ download_path = os.path.join(download_dir, link.filename)
+ if os.path.exists(download_path):
+ content_type = mimetypes.guess_type(download_path)[0]
+ logger.notify('File was already downloaded %s' % download_path)
+ if link.hash:
+ download_hash = _get_hash_from_file(download_path, link)
+ try:
+ _check_hash(download_hash, link)
+ already_downloaded = True
+ except HashMismatch:
+ logger.warn(
+ 'Previously-downloaded file %s has bad hash, '
+ 're-downloading.' % link_path
+ )
+ os.unlink(download_path)
+ else:
+ already_downloaded = True
+
+ # a download dir is specified and not already downloaded
+ if download_dir and not already_downloaded:
+ content_type = mimetypes.guess_type(link_path)[0]
+ _copy_file(link_path, download_dir, content_type, link)
+
+ # unpack the archive to the build dir location. even when only downloading
+ # archives, they have to be unpacked to parse dependencies
+ if already_downloaded:
+ from_path = download_path
+ else:
+ from_path = link_path
+ content_type = mimetypes.guess_type(from_path)[0]
+ unpack_file(from_path, location, content_type, link)
diff --git a/pip/req/req_set.py b/pip/req/req_set.py
index e398e5072af..3391e6f7d64 100644
--- a/pip/req/req_set.py
+++ b/pip/req/req_set.py
@@ -9,14 +9,14 @@
from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled,
DistributionNotFound, PreviousBuildDirError)
from pip.index import Link
-from pip.locations import (
- PIP_DELETE_MARKER_FILENAME, write_delete_marker_file, build_prefix,
-)
+from pip.locations import (PIP_DELETE_MARKER_FILENAME, build_prefix,
+ write_delete_marker_file)
from pip.log import logger
from pip.req.req_install import InstallRequirement
from pip.util import (display_path, rmtree, dist_in_usersite, call_subprocess,
_make_build_dir)
from pip.vcs import vcs
+from pip.wheel import wheel_ext
class Requirements(object):
@@ -53,10 +53,12 @@ def __init__(self, build_dir, src_dir, download_dir, download_cache=None,
upgrade=False, ignore_installed=False, as_egg=False,
target_dir=None, ignore_dependencies=False,
force_reinstall=False, use_user_site=False, session=None,
- pycompile=True):
+ pycompile=True, wheel_download_dir=None):
self.build_dir = build_dir
self.src_dir = src_dir
self.download_dir = download_dir
+ if download_cache:
+ download_cache = os.path.expanduser(download_cache)
self.download_cache = download_cache
self.upgrade = upgrade
self.ignore_installed = ignore_installed
@@ -74,6 +76,7 @@ def __init__(self, build_dir, src_dir, download_dir, download_cache=None,
self.target_dir = target_dir # set from --target option
self.session = session or PipSession()
self.pycompile = pycompile
+ self.wheel_download_dir = wheel_download_dir
def __str__(self):
reqs = [req for req in self.requirements.values()
@@ -209,6 +212,11 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
install = True
best_installed = False
not_found = None
+
+ ###############################################
+ ## Search for archive to fulfill requirement ##
+ ###############################################
+
if not self.ignore_installed and not req_to_install.editable:
req_to_install.check_if_exists()
if req_to_install.satisfied_by:
@@ -258,6 +266,11 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
else:
logger.notify('Downloading/unpacking %s' % req_to_install)
logger.indent += 2
+
+ ##################################
+ ## vcs update or unpack archive ##
+ ##################################
+
try:
is_bundle = False
is_wheel = False
@@ -323,9 +336,21 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
assert url
if url:
try:
+
+ if (
+ url.filename.endswith(wheel_ext)
+ and self.wheel_download_dir
+ ):
+ # when doing 'pip wheel`
+ download_dir = self.wheel_download_dir
+ do_download = True
+ else:
+ download_dir = self.download_dir
+ do_download = self.is_download
self.unpack_url(
- url, location, self.is_download,
- )
+ url, location, download_dir,
+ do_download,
+ )
except HTTPError as exc:
logger.fatal(
'Could not install requirement %s because '
@@ -340,7 +365,7 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
unpack = False
if unpack:
is_bundle = req_to_install.is_bundle
- is_wheel = url and url.filename.endswith('.whl')
+ is_wheel = url and url.filename.endswith(wheel_ext)
if is_bundle:
req_to_install.move_bundle_files(
self.build_dir,
@@ -356,6 +381,11 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
req_to_install.run_egg_info()
if url and url.scheme in vcs.all_schemes:
req_to_install.archive(self.download_dir)
+
+ ##############################
+ ## parse wheel dependencies ##
+ ##############################
+
elif is_wheel:
req_to_install.source_dir = location
req_to_install.url = url.url
@@ -413,6 +443,11 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False):
req_to_install
)
install = False
+
+ ##############################
+ ## parse sdist dependencies ##
+ ##############################
+
if not (is_bundle or is_wheel):
## FIXME: shouldn't be globally added:
finder.add_dependency_links(
@@ -503,29 +538,36 @@ def copy_to_build_dir(self, req_to_install):
call_subprocess(["python", "%s/setup.py" % dest, "clean"], cwd=dest,
command_desc='python setup.py clean')
- def unpack_url(self, link, location, only_download=False):
- if only_download:
- loc = self.download_dir
- else:
- loc = location
+ def unpack_url(self, link, location, download_dir=None,
+ only_download=False):
+ if download_dir is None:
+ download_dir = self.download_dir
+
+ # non-editable vcs urls
if is_vcs_url(link):
- return unpack_vcs_link(link, loc, only_download)
- # a local file:// index could have links with hashes
- elif not link.hash and is_file_url(link):
- return unpack_file_url(link, loc)
+ if only_download:
+ loc = download_dir
+ else:
+ loc = location
+ unpack_vcs_link(link, loc, only_download)
+
+ # file urls
+ elif is_file_url(link):
+ unpack_file_url(link, location, download_dir)
+ if only_download:
+ write_delete_marker_file(location)
+
+ # http urls
else:
- if self.download_cache:
- self.download_cache = os.path.expanduser(self.download_cache)
- retval = unpack_http_url(
+ unpack_http_url(
link,
location,
self.download_cache,
- self.download_dir,
+ download_dir,
self.session,
)
if only_download:
write_delete_marker_file(location)
- return retval
def install(self, install_options, global_options=(), *args, **kwargs):
"""
diff --git a/pip/wheel.py b/pip/wheel.py
index 849cb41b591..58148a09f33 100644
--- a/pip/wheel.py
+++ b/pip/wheel.py
@@ -487,21 +487,19 @@ def build(self):
reqset = self.requirement_set.requirements.values()
- #make the wheelhouse
- if not os.path.exists(self.wheel_dir):
- os.makedirs(self.wheel_dir)
+ buildset = [req for req in reqset if not req.is_wheel]
+
+ if not buildset:
+ return
#build the wheels
logger.notify(
'Building wheels for collected packages: %s' %
- ','.join([req.name for req in reqset])
+ ','.join([req.name for req in buildset])
)
logger.indent += 2
build_success, build_failure = [], []
- for req in reqset:
- if req.is_wheel:
- logger.notify("Skipping building wheel: %s", req.url)
- continue
+ for req in buildset:
if self._build_one(req):
build_success.append(req)
else:
diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py
index f7344e81258..98e61db3489 100644
--- a/tests/functional/test_wheel.py
+++ b/tests/functional/test_wheel.py
@@ -36,6 +36,20 @@ def test_pip_wheel_success(script, data):
assert "Successfully built simple" in result.stdout, result.stdout
+def test_pip_wheel_downloads_wheels(script, data):
+ """
+ Test 'pip wheel' downloads wheels
+ """
+ script.pip('install', 'wheel')
+ result = script.pip(
+ 'wheel', '--no-index', '-f', data.find_links, 'simple.dist',
+ )
+ wheel_file_name = 'simple.dist-0.1-py2.py3-none-any.whl'
+ wheel_file_path = script.scratch/'wheelhouse'/wheel_file_name
+ assert wheel_file_path in result.files_created, result.stdout
+ assert "Saved" in result.stdout, result.stdout
+
+
def test_pip_wheel_fail(script, data):
"""
Test 'pip wheel' failure.
@@ -56,7 +70,7 @@ def test_pip_wheel_fail(script, data):
def test_pip_wheel_ignore_wheels_editables(script, data):
"""
- Test 'pip wheel' ignores editables and *.whl files in requirements
+ Test 'pip wheel' ignores editables
"""
script.pip('install', 'wheel')
@@ -79,7 +93,6 @@ def test_pip_wheel_ignore_wheels_editables(script, data):
)
assert "Successfully built simple" in result.stdout, result.stdout
assert "Failed to build" not in result.stdout, result.stdout
- assert "ignoring %s" % local_wheel in result.stdout
ignore_editable = "ignoring %s" % path_to_url(local_editable)
#TODO: understand this divergence
if sys.platform == 'win32':
diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py
index 8dcfdfa5078..5ee5b2953bf 100644
--- a/tests/unit/test_download.py
+++ b/tests/unit/test_download.py
@@ -1,6 +1,6 @@
import hashlib
import os
-from shutil import rmtree
+from shutil import rmtree, copy
from tempfile import mkdtemp
from mock import Mock, patch
@@ -8,7 +8,8 @@
import pip
from pip.backwardcompat import urllib, BytesIO, b, pathname2url
-from pip.download import PipSession, path_to_url, unpack_http_url, url_to_path
+from pip.download import (PipSession, path_to_url, unpack_http_url,
+ url_to_path, unpack_file_url)
from pip.index import Link
@@ -182,3 +183,89 @@ def test_path_to_url_win():
@pytest.mark.skipif("sys.platform != 'win32'")
def test_url_to_path_win():
assert url_to_path('file:///c:/tmp/file') == 'c:/tmp/file'
+
+
+class Test_unpack_file_url(object):
+
+ def prep(self, tmpdir, data):
+ self.build_dir = tmpdir.join('build')
+ self.download_dir = tmpdir.join('download')
+ os.mkdir(self.build_dir)
+ os.mkdir(self.download_dir)
+ self.dist_file = "simple-1.0.tar.gz"
+ self.dist_file2 = "simple-2.0.tar.gz"
+ self.dist_path = data.packages.join(self.dist_file)
+ self.dist_path2 = data.packages.join(self.dist_file2)
+ self.dist_url = Link(path_to_url(self.dist_path))
+ self.dist_url2 = Link(path_to_url(self.dist_path2))
+
+ def test_unpack_file_url_no_download(self, tmpdir, data):
+ self.prep(tmpdir, data)
+ unpack_file_url(self.dist_url, self.build_dir)
+ assert os.path.isdir(os.path.join(self.build_dir, 'simple'))
+ assert not os.path.isfile(
+ os.path.join(self.download_dir, self.dist_file))
+
+ def test_unpack_file_url_and_download(self, tmpdir, data):
+ self.prep(tmpdir, data)
+ unpack_file_url(self.dist_url, self.build_dir,
+ download_dir=self.download_dir)
+ assert os.path.isdir(os.path.join(self.build_dir, 'simple'))
+ assert os.path.isfile(os.path.join(self.download_dir, self.dist_file))
+
+ def test_unpack_file_url_download_already_exists(self, tmpdir,
+ data, monkeypatch):
+ self.prep(tmpdir, data)
+ # add in previous download (copy simple-2.0 as simple-1.0)
+ # so we can tell it didn't get overwritten
+ dest_file = os.path.join(self.download_dir, self.dist_file)
+ copy(self.dist_path2, dest_file)
+ dist_path2_md5 = hashlib.md5(
+ open(self.dist_path2, 'rb').read()).hexdigest()
+
+ unpack_file_url(self.dist_url, self.build_dir,
+ download_dir=self.download_dir)
+ # our hash should be the same, i.e. not overwritten by simple-1.0 hash
+ assert dist_path2_md5 == hashlib.md5(
+ open(dest_file, 'rb').read()).hexdigest()
+
+ def test_unpack_file_url_download_bad_hash(self, tmpdir, data,
+ monkeypatch):
+ """
+ Test when existing download has different hash from the file url
+ fragment
+ """
+ self.prep(tmpdir, data)
+
+ # add in previous download (copy simple-2.0 as simple-1.0 so it's wrong
+ # hash)
+ dest_file = os.path.join(self.download_dir, self.dist_file)
+ copy(self.dist_path2, dest_file)
+
+ dist_path_md5 = hashlib.md5(
+ open(self.dist_path, 'rb').read()).hexdigest()
+ dist_path2_md5 = hashlib.md5(open(dest_file, 'rb').read()).hexdigest()
+
+ assert dist_path_md5 != dist_path2_md5
+
+ self.dist_url.url = "%s#md5=%s" % (
+ self.dist_url.url,
+ dist_path_md5
+ )
+ unpack_file_url(self.dist_url, self.build_dir,
+ download_dir=self.download_dir)
+
+ # confirm hash is for simple1-1.0
+ # the previous bad download has been removed
+ assert (hashlib.md5(open(dest_file, 'rb').read()).hexdigest()
+ ==
+ dist_path_md5
+ ), hashlib.md5(open(dest_file, 'rb').read()).hexdigest()
+
+ def test_unpack_file_url_thats_a_dir(self, tmpdir, data):
+ self.prep(tmpdir, data)
+ dist_path = data.packages.join("FSPkg")
+ dist_url = Link(path_to_url(dist_path))
+ unpack_file_url(dist_url, self.build_dir,
+ download_dir=self.download_dir)
+ assert os.path.isdir(os.path.join(self.build_dir, 'fspkg'))
From eb7a31e01961130a476abea4b564c51bd63c64ee Mon Sep 17 00:00:00 2001
From: Marcus Smith <qwcode@gmail.com>
Date: Sat, 1 Feb 2014 14:04:58 -0800
Subject: [PATCH 2/2] when file urls have hash fragments, check it
---
pip/download.py | 21 +++++++++++++--------
tests/unit/test_download.py | 11 +++++++++++
2 files changed, 24 insertions(+), 8 deletions(-)
diff --git a/pip/download.py b/pip/download.py
index a9169842acd..7458e48e9d3 100644
--- a/pip/download.py
+++ b/pip/download.py
@@ -625,7 +625,6 @@ def unpack_http_url(link, location, download_cache, download_dir=None,
def unpack_file_url(link, location, download_dir=None):
link_path = url_to_path(link.url_without_fragment)
- from_path = None
already_downloaded = False
# If it's a url to a local directory
@@ -635,6 +634,11 @@ def unpack_file_url(link, location, download_dir=None):
shutil.copytree(link_path, location, symlinks=True)
return
+ # if link has a hash, let's confirm it matches
+ if link.hash:
+ link_path_hash = _get_hash_from_file(link_path, link)
+ _check_hash(link_path_hash, link)
+
# If a download dir is specified, is the file already there and valid?
if download_dir:
download_path = os.path.join(download_dir, link.filename)
@@ -655,16 +659,17 @@ def unpack_file_url(link, location, download_dir=None):
else:
already_downloaded = True
- # a download dir is specified and not already downloaded
- if download_dir and not already_downloaded:
- content_type = mimetypes.guess_type(link_path)[0]
- _copy_file(link_path, download_dir, content_type, link)
-
- # unpack the archive to the build dir location. even when only downloading
- # archives, they have to be unpacked to parse dependencies
if already_downloaded:
from_path = download_path
else:
from_path = link_path
+
content_type = mimetypes.guess_type(from_path)[0]
+
+ # unpack the archive to the build dir location. even when only downloading
+ # archives, they have to be unpacked to parse dependencies
unpack_file(from_path, location, content_type, link)
+
+ # a download dir is specified and not already downloaded
+ if download_dir and not already_downloaded:
+ _copy_file(from_path, download_dir, content_type, link)
diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py
index 5ee5b2953bf..e79eaecb21f 100644
--- a/tests/unit/test_download.py
+++ b/tests/unit/test_download.py
@@ -8,6 +8,7 @@
import pip
from pip.backwardcompat import urllib, BytesIO, b, pathname2url
+from pip.exceptions import HashMismatch
from pip.download import (PipSession, path_to_url, unpack_http_url,
url_to_path, unpack_file_url)
from pip.index import Link
@@ -229,6 +230,16 @@ def test_unpack_file_url_download_already_exists(self, tmpdir,
assert dist_path2_md5 == hashlib.md5(
open(dest_file, 'rb').read()).hexdigest()
+ def test_unpack_file_url_bad_hash(self, tmpdir, data,
+ monkeypatch):
+ """
+ Test when the file url hash fragment is wrong
+ """
+ self.prep(tmpdir, data)
+ self.dist_url.url = "%s#md5=bogus" % self.dist_url.url
+ with pytest.raises(HashMismatch):
+ unpack_file_url(self.dist_url, self.build_dir)
+
def test_unpack_file_url_download_bad_hash(self, tmpdir, data,
monkeypatch):
"""