Feature/better resolver

pipenv/_compat.py

@@ -29,6 +29,11 @@ def _infer_return_type(*args):
                 _types.add(type(arg))
         return _types.pop()
 
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
+
 
 try:
     from weakref import finalize
@@ -51,8 +56,6 @@ def detach(self):
     class ResourceWarning(Warning):
         pass
 
-# -*- coding=utf-8 -*-
-
 
 def pip_import(module_path, subimport=None, old_path=None):
     internal = 'pip._internal.{0}'.format(module_path)

pipenv/core.py

@@ -46,10 +46,12 @@
     rmtree,
     split_argument,
     extract_uri_from_vcs_dep,
+    fs_str,
 )
 from ._compat import (
     TemporaryDirectory,
-    vcs
+    vcs,
+    Path
 )
 from .import pep508checker, progress
 from .environments import (
@@ -73,6 +75,7 @@
     PIPENV_SHELL,
     PIPENV_PYTHON,
     PIPENV_VIRTUALENV,
+    PIPENV_CACHE_DIR,
 )
 
 # Backport required for earlier versions of Python.
@@ -1481,7 +1484,7 @@ def pip_install(
     pre = '--pre' if pre else ''
     quoted_pip = which_pip(allow_global=allow_global)
     quoted_pip = escape_grouped_arguments(quoted_pip)
-    upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else ''
+    upgrade_strategy = '--upgrade --upgrade-strategy=to-satisfy-only' if selective_upgrade else ''
     pip_command = '{0} install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format(
         quoted_pip,
         install_reqs,
@@ -1494,19 +1497,31 @@ def pip_install(
     )
     if verbose:
         click.echo('$ {0}'.format(pip_command), err=True)
-    c = delegator.run(pip_command, block=block)
+    cache_dir = Path(PIPENV_CACHE_DIR)
+    pip_config = {
+        'PIP_CACHE_DIR': fs_str(cache_dir.as_posix()),
+        'PIP_WHEEL_DIR': fs_str(cache_dir.joinpath('wheels').as_posix()),
+        'PIP_DESTINATION_DIR': fs_str(cache_dir.joinpath('pkgs').as_posix()),
+    }
+    c = delegator.run(pip_command, block=block, env=pip_config)
     return c
 
 
 def pip_download(package_name):
+    cache_dir = Path(PIPENV_CACHE_DIR)    
+    pip_config = {
+        'PIP_CACHE_DIR': fs_str(cache_dir.as_posix()),
+        'PIP_WHEEL_DIR': fs_str(cache_dir.joinpath('wheels').as_posix()),
+        'PIP_DESTINATION_DIR': fs_str(cache_dir.joinpath('pkgs').as_posix()),
+    }
     for source in project.sources:
         cmd = '{0} download "{1}" -i {2} -d {3}'.format(
             escape_grouped_arguments(which_pip()),
             package_name,
             source['url'],
             project.download_location,
         )
-        c = delegator.run(cmd)
+        c = delegator.run(cmd, env=pip_config)
         if c.return_code == 0:
             break
 

pipenv/patched/piptools/repositories/pypi.py

@@ -4,6 +4,7 @@
 
 import hashlib
 import os
+import sys
 from contextlib import contextmanager
 from shutil import rmtree
 
@@ -20,14 +21,17 @@
     SafeFileCache,
 )
 
-from notpip._vendor.packaging.requirements import InvalidRequirement
+from notpip._vendor.packaging.requirements import InvalidRequirement, Requirement
+from notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
+from notpip._vendor.packaging.specifiers import SpecifierSet
 from notpip._vendor.pyparsing import ParseException
 
 from ..cache import CACHE_DIR
 from pipenv.environments import PIPENV_CACHE_DIR
 from ..exceptions import NoCandidateFound
-from ..utils import (fs_str, is_pinned_requirement, lookup_table,
-                     make_install_requirement)
+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
+                     make_install_requirement, format_requirement, dedup)
+
 from .base import BaseRepository
 
 
@@ -159,7 +163,13 @@ def find_best_match(self, ireq, prereleases=None):
         if ireq.editable:
             return ireq  # return itself as the best match
 
-        all_candidates = self.find_all_candidates(ireq.name)
+        py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
+        all_candidates = []
+        for c in self.find_all_candidates(ireq.name):
+            if c.requires_python and not SpecifierSet(c.requires_python).contains(py_version):
+                continue
+            all_candidates.append(c)
+
         candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
         try:
             matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
@@ -188,21 +198,33 @@ def get_json_dependencies(self, ireq):
             raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq))
 
         def gen(ireq):
-            if self.DEFAULT_INDEX_URL in self.finder.index_urls:
-
-                url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name)
-                r = self.session.get(url)
-
-                # TODO: Latest isn't always latest.
-                latest = list(r.json()['releases'].keys())[-1]
-                if str(ireq.req.specifier) == '=={0}'.format(latest):
-                    latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest)
-                    latest_requires = self.session.get(latest_url)
-                    for requires in latest_requires.json().get('info', {}).get('requires_dist', {}):
-                        i = InstallRequirement.from_line(requires)
+            if self.DEFAULT_INDEX_URL not in self.finder.index_urls:
+                return
+
+            url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name)
+            releases = self.session.get(url).json()['releases']
+
+            matches = [
+                r for r in releases
+                if '=={0}'.format(r) == str(ireq.req.specifier)
+            ]
+            if not matches:
+                return
+
+            release_requires = self.session.get(
+                'https://pypi.org/pypi/{0}/{1}/json'.format(
+                    ireq.req.name, matches[0],
+                ),
+            ).json()
+            try:
+                requires_dist = release_requires['info']['requires_dist']
+            except KeyError:
+                return
 
-                        if 'extra' not in repr(i.markers):
-                            yield i
+            for requires in requires_dist:
+                i = InstallRequirement.from_line(requires)
+                if 'extra' not in repr(i.markers):
+                    yield i
 
         try:
             if ireq not in self._json_dep_cache:
@@ -226,7 +248,6 @@ def get_dependencies(self, ireq):
 
         return json_results
 
-
     def get_legacy_dependencies(self, ireq):
         """
         Given a pinned or an editable InstallRequirement, returns a set of
@@ -245,7 +266,13 @@ def get_legacy_dependencies(self, ireq):
                     setup_requires = self.finder.get_extras_links(
                         dist.get_metadata_lines('requires.txt')
                     )
-            except TypeError:
+                # HACK: Sometimes the InstallRequirement doesn't properly get
+                # these values set on it during the resolution process. It's
+                # difficult to pin down what is going wrong. This fixes things.
+                ireq.version = dist.version
+                ireq.project_name = dist.project_name
+                ireq.req = dist.as_requirement()
+            except (TypeError, ValueError):
                 pass
 
         if ireq not in self._dependencies_cache:

pipenv/project.py

@@ -20,7 +20,7 @@
     from pathlib2 import Path
 
 from .cmdparse import Script
-from .vendor.requirementslib import Requirement
+from .vendor.requirementslib.requirements import Requirement
 from .utils import (
     atomic_open_for_write,
     mkdir_p,
@@ -728,8 +728,8 @@ def add_package_to_pipfile(self, package_name, dev=False):
         # Read and append Pipfile.
         p = self.parsed_pipfile
         # Don't re-capitalize file URLs or VCSs.
-        package = Requirement.from_line(package_name)
-        converted = first(package.as_pipfile().values())
+        package = Requirement.from_line(package_name.strip())
+        _, converted = package.pipfile_entry
         key = 'dev-packages' if dev else 'packages'
         # Set empty group if it doesn't exist yet.
         if key not in p: