Merge pull request #2935 from pypa/update-pip-piptools

Update pip and piptools to fix pip 18 upgrades

MANIFEST.in

@@ -8,6 +8,7 @@ recursive-include pipenv *.md *.APACHE *.BSD
 recursive-include pipenv Makefile
 recursive-include pipenv/vendor vendor.txt
 recursive-include pipenv README
+recursive-include pipenv *.json
 include pipenv/patched/notpip/_vendor/vendor.txt
 include pipenv/patched/safety.zip pipenv/patched/patched.txt
 include pipenv/vendor/pipreqs/stdlib pipenv/vendor/pipreqs/mapping
@@ -23,7 +24,8 @@ recursive-include docs Makefile *.rst *.py *.bat
 recursive-include docs/_templates *.html
 recursive-include docs/_static *.js *.css *.png
 recursive-exclude docs requirements*.txt
-
+recursive-exclude pipenv *.pyi
+recursive-exclude pipenv *.typed
 
 prune peeps
 prune .buildkite

news/2499.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2529.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2589.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2666.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2767.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2785.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2795.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2801.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2802.bugfix

@@ -0,0 +1,19 @@
+Updated vendored dependencies:
+ - ``pip-tools`` (updated and patched to latest w/ ``pip 18.0`` compatibilty)
+ - ``pip 10.0.1 => 18.0``
+ - ``click 6.7 => 7.0``
+ - ``toml 0.9.4 => 0.10.0``
+ - ``pyparsing 2.2.0 => 2.2.2``
+ - ``delegator 0.1.0 => 0.1.1``
+ - ``attrs 18.1.0 => 18.2.0``
+ - ``distlib 0.2.7 => 0.2.8``
+ - ``packaging 17.1.0 => 18.0``
+ - ``passa 0.2.0 => 0.3.1``
+ - ``pip_shims 0.1.2 => 0.3.1``
+ - ``plette 0.1.1 => 0.2.2``
+ - ``pythonfinder 1.0.2 => 1.1.0``
+ - ``pytoml 0.1.18 => 0.1.19``
+ - ``requirementslib 1.1.16 => 1.1.17``
+ - ``shellingham 1.2.4 => 1.2.6``
+ - ``tomlkit 0.4.2 => 0.4.4``
+ - ``vistir 0.1.4 => 0.1.6``

news/2824.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2862.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2867.bugfix

@@ -0,0 +1,19 @@
+Updated vendored dependencies:
+ - ``pip-tools`` (updated and patched to latest w/ ``pip 18.0`` compatibilty)
+ - ``pip 10.0.1 => 18.0``
+ - ``click 6.7 => 7.0``
+ - ``toml 0.9.4 => 0.10.0``
+ - ``pyparsing 2.2.0 => 2.2.2``
+ - ``delegator 0.1.0 => 0.1.1``
+ - ``attrs 18.1.0 => 18.2.0``
+ - ``distlib 0.2.7 => 0.2.8``
+ - ``packaging 17.1.0 => 18.0``
+ - ``passa 0.2.0 => 0.3.1``
+ - ``pip_shims 0.1.2 => 0.3.1``
+ - ``plette 0.1.1 => 0.2.2``
+ - ``pythonfinder 1.0.2 => 1.1.0``
+ - ``pytoml 0.1.18 => 0.1.19``
+ - ``requirementslib 1.1.16 => 1.1.17``
+ - ``shellingham 1.2.4 => 1.2.6``
+ - ``tomlkit 0.4.2 => 0.4.4``
+ - ``vistir 0.1.4 => 0.1.6``

news/2879.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2880.bugfix

@@ -0,0 +1,19 @@
+Updated vendored dependencies:
+ - ``pip-tools`` (updated and patched to latest w/ ``pip 18.0`` compatibilty)
+ - ``pip 10.0.1 => 18.0``
+ - ``click 6.7 => 7.0``
+ - ``toml 0.9.4 => 0.10.0``
+ - ``pyparsing 2.2.0 => 2.2.2``
+ - ``delegator 0.1.0 => 0.1.1``
+ - ``attrs 18.1.0 => 18.2.0``
+ - ``distlib 0.2.7 => 0.2.8``
+ - ``packaging 17.1.0 => 18.0``
+ - ``passa 0.2.0 => 0.3.1``
+ - ``pip_shims 0.1.2 => 0.3.1``
+ - ``plette 0.1.1 => 0.2.2``
+ - ``pythonfinder 1.0.2 => 1.1.0``
+ - ``pytoml 0.1.18 => 0.1.19``
+ - ``requirementslib 1.1.16 => 1.1.17``
+ - ``shellingham 1.2.4 => 1.2.6``
+ - ``tomlkit 0.4.2 => 0.4.4``
+ - ``vistir 0.1.4 => 0.1.6``

news/2894.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2902.vendor

@@ -0,0 +1,19 @@
+Updated vendored dependencies:
+ - ``pip-tools`` (updated and patched to latest w/ ``pip 18.0`` compatibilty)
+ - ``pip 10.0.1 => 18.0``
+ - ``click 6.7 => 7.0``
+ - ``toml 0.9.4 => 0.10.0``
+ - ``pyparsing 2.2.0 => 2.2.2``
+ - ``delegator 0.1.0 => 0.1.1``
+ - ``attrs 18.1.0 => 18.2.0``
+ - ``distlib 0.2.7 => 0.2.8``
+ - ``packaging 17.1.0 => 18.0``
+ - ``passa 0.2.0 => 0.3.1``
+ - ``pip_shims 0.1.2 => 0.3.1``
+ - ``plette 0.1.1 => 0.2.2``
+ - ``pythonfinder 1.0.2 => 1.1.0``
+ - ``pytoml 0.1.18 => 0.1.19``
+ - ``requirementslib 1.1.16 => 1.1.17``
+ - ``shellingham 1.2.4 => 1.2.6``
+ - ``tomlkit 0.4.2 => 0.4.4``
+ - ``vistir 0.1.4 => 0.1.6``

news/2924.bugfix

@@ -0,0 +1 @@
+Fixed various bugs related to ``pip 18.1`` release which prevented locking, installation, and syncing, and dumping to a ``requirements.txt`` file.

news/2933.bugfix

@@ -0,0 +1 @@
+Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``.

news/2935.vendor

@@ -0,0 +1,19 @@
+Updated vendored dependencies:
+ - ``pip-tools`` (updated and patched to latest w/ ``pip 18.0`` compatibilty)
+ - ``pip 10.0.1 => 18.0``
+ - ``click 6.7 => 7.0``
+ - ``toml 0.9.4 => 0.10.0``
+ - ``pyparsing 2.2.0 => 2.2.2``
+ - ``delegator 0.1.0 => 0.1.1``
+ - ``attrs 18.1.0 => 18.2.0``
+ - ``distlib 0.2.7 => 0.2.8``
+ - ``packaging 17.1.0 => 18.0``
+ - ``passa 0.2.0 => 0.3.1``
+ - ``pip_shims 0.1.2 => 0.3.1``
+ - ``plette 0.1.1 => 0.2.2``
+ - ``pythonfinder 1.0.2 => 1.1.0``
+ - ``pytoml 0.1.18 => 0.1.19``
+ - ``requirementslib 1.1.16 => 1.1.17``
+ - ``shellingham 1.2.4 => 1.2.6``
+ - ``tomlkit 0.4.2 => 0.4.4``
+ - ``vistir 0.1.4 => 0.1.6``

pipenv/core.py

@@ -14,6 +14,7 @@
 import delegator
 import pipfile
 from blindspin import spinner
+import vistir
 import six
 
 from .cmdparse import Script
@@ -37,10 +38,9 @@
  is_pinned,
  is_star,
  rmtree,
- fs_str,
  clean_resolved_dep,
+ parse_indexes,
 )
-from ._compat import TemporaryDirectory, Path
 from . import environments, pep508checker, progress
 from .environments import (
  PIPENV_COLORBLIND,
@@ -67,9 +67,7 @@
  "wheel",
 )
 
-FIRST_PACKAGES = (
- "cython",
-)
+FIRST_PACKAGES = ("cython",)
 # Are we using the default Python?
 USING_DEFAULT_PYTHON = True
 if not PIPENV_HIDE_EMOJIS:
@@ -215,10 +213,14 @@ def import_requirements(r=None, dev=False):
  with open(r, "r") as f:
  contents = f.read()
  indexes = []
+ trusted_hosts = []
  # Find and add extra indexes.
  for line in contents.split("\n"):
- if line.startswith(("-i ", "--index ", "--index-url ")):
- indexes.append(line.split()[1])
+ line_indexes, _trusted_hosts, _ = parse_indexes(line.strip())
+ indexes.extend(line_indexes)
+ trusted_hosts.extend(_trusted_hosts)
+ indexes = sorted(set(indexes))
+ trusted_hosts = sorted(set(trusted_hosts))
  reqs = [f for f in parse_requirements(r, session=pip_requests)]
  for package in reqs:
  if package.name not in BAD_PACKAGES:
@@ -232,7 +234,8 @@ def import_requirements(r=None, dev=False):
  else:
  project.add_package_to_pipfile(str(package.req), dev=dev)
  for index in indexes:
- project.add_index_to_pipfile(index)
+ trusted = index in trusted_hosts
+ project.add_index_to_pipfile(index, trusted_host=trusted)
  project.recase_pipfile()
 
 
@@ -741,17 +744,14 @@ def cleanup_procs(procs, concurrent):
  for dep, ignore_hash, block in deps_list_bar:
  if len(procs) < PIPENV_MAX_SUBPROCESS:
  # Use a specific index, if specified.
+ indexes, trusted_hosts, dep = parse_indexes(dep)
  index = None
- if " --index" in dep:
- dep, _, index = dep.partition(" --index")
- index = index.lstrip("=")
- elif " -i " in dep:
- dep, _, index = dep.partition(" -i ")
  extra_indexes = []
- if "--extra-index-url" in dep:
- split_dep = dep.split("--extra-index-url")
- dep, extra_indexes = split_dep[0], split_dep[1:]
- dep = Requirement.from_line(dep)
+ if indexes:
+ index = indexes[0]
+ if len(indexes) > 0:
+ extra_indexes = indexes[1:]
+ dep = Requirement.from_line(" ".join(dep))
  if index:
  _index = None
  try:
@@ -883,7 +883,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None):
  cmd.append("--system-site-packages")
 
  if pypi_mirror:
- pip_config = {"PIP_INDEX_URL": fs_str(pypi_mirror)}
+ pip_config = {"PIP_INDEX_URL": vistir.misc.fs_str(pypi_mirror)}
  else:
  pip_config = {}
 
@@ -906,7 +906,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None):
  # This mimics Pew's "setproject".
  project_file_name = os.path.join(project.virtualenv_location, ".project")
  with open(project_file_name, "w") as f:
- f.write(fs_str(project.project_directory))
+ f.write(vistir.misc.fs_str(project.project_directory))
 
  # Say where the virtualenv is.
  do_where(virtualenv=True, bare=False)
@@ -1193,7 +1193,9 @@ def do_init(
  ensure_pipfile(system=system)
  if not requirements_dir:
  cleanup_reqdir = True
- requirements_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-")
+ requirements_dir = vistir.compat.TemporaryDirectory(
+ suffix="-requirements", prefix="pipenv-"
+ )
  # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored
  if (project.lockfile_exists and not ignore_pipfile) and not skip_lock:
  old_hash = project.get_lockfile_hash()
@@ -1386,27 +1388,33 @@ def pip_install(
 
  if environments.is_verbose():
  click.echo("$ {0}".format(pip_command), err=True)
- cache_dir = Path(PIPENV_CACHE_DIR)
+ cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR)
  pip_config = {
- "PIP_CACHE_DIR": fs_str(cache_dir.as_posix()),
- "PIP_WHEEL_DIR": fs_str(cache_dir.joinpath("wheels").as_posix()),
- "PIP_DESTINATION_DIR": fs_str(cache_dir.joinpath("pkgs").as_posix()),
- "PIP_EXISTS_ACTION": fs_str("w"),
- "PATH": fs_str(os.environ.get("PATH")),
+ "PIP_CACHE_DIR": vistir.misc.fs_str(cache_dir.as_posix()),
+ "PIP_WHEEL_DIR": vistir.misc.fs_str(cache_dir.joinpath("wheels").as_posix()),
+ "PIP_DESTINATION_DIR": vistir.misc.fs_str(
+ cache_dir.joinpath("pkgs").as_posix()
+ ),
+ "PIP_EXISTS_ACTION": vistir.misc.fs_str("w"),
+ "PATH": vistir.misc.fs_str(os.environ.get("PATH")),
  }
  if src:
- pip_config.update({"PIP_SRC": fs_str(project.virtualenv_src_location)})
+ pip_config.update(
+ {"PIP_SRC": vistir.misc.fs_str(project.virtualenv_src_location)}
+ )
  pip_command = Script.parse(pip_command).cmdify()
  c = delegator.run(pip_command, block=block, env=pip_config)
  return c
 
 
 def pip_download(package_name):
- cache_dir = Path(PIPENV_CACHE_DIR)
+ cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR)
  pip_config = {
- "PIP_CACHE_DIR": fs_str(cache_dir.as_posix()),
- "PIP_WHEEL_DIR": fs_str(cache_dir.joinpath("wheels").as_posix()),
- "PIP_DESTINATION_DIR": fs_str(cache_dir.joinpath("pkgs").as_posix()),
+ "PIP_CACHE_DIR": vistir.misc.fs_str(cache_dir.as_posix()),
+ "PIP_WHEEL_DIR": vistir.misc.fs_str(cache_dir.joinpath("wheels").as_posix()),
+ "PIP_DESTINATION_DIR": vistir.misc.fs_str(
+ cache_dir.joinpath("pkgs").as_posix()
+ ),
  }
  for source in project.sources:
  cmd = '{0} download "{1}" -i {2} -d {3}'.format(
@@ -1663,7 +1671,7 @@ def do_install(
  from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM
  from notpip._internal.exceptions import PipError
 
- requirements_directory = TemporaryDirectory(
+ requirements_directory = vistir.compat.TemporaryDirectory(
  suffix="-requirements", prefix="pipenv-"
  )
  if selective_upgrade:
@@ -2034,7 +2042,7 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror
  # Ensure that virtualenv is available.
  ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror)
  # Set an environment variable, so we know we're in the environment.
- os.environ["PIPENV_ACTIVE"] = "1"
+ os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1")
  # Support shell compatibility mode.
  if PIPENV_SHELL_FANCY:
  fancy = True
@@ -2111,7 +2119,7 @@ def inline_activate_virtual_environment():
  else:
  _inline_activate_virtualenv()
  if "VIRTUAL_ENV" not in os.environ:
- os.environ["VIRTUAL_ENV"] = root
+ os.environ["VIRTUAL_ENV"] = vistir.misc.fs_str(root)
 
 
 def _launch_windows_subprocess(script):
@@ -2468,7 +2476,9 @@ def do_sync(
  )
 
  # Install everything.
- requirements_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-")
+ requirements_dir = vistir.compat.TemporaryDirectory(
+ suffix="-requirements", prefix="pipenv-"
+ )
  do_init(
  dev=dev,
  concurrent=(not sequential),

pipenv/environments.py

@@ -1,18 +1,19 @@
 import os
 import sys
 from appdirs import user_cache_dir
+from .vendor.vistir.misc import fs_str
 
 
 # HACK: avoid resolver.py uses the wrong byte code files.
 # I hope I can remove this one day.
-os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
+os.environ["PYTHONDONTWRITEBYTECODE"] = fs_str("1")
 
 # HACK: Prevent invalid shebangs with Homebrew-installed Python:
 # https://bugs.python.org/issue22490
 os.environ.pop("__PYVENV_LAUNCHER__", None)
 
 # Load patched pip instead of system pip
-os.environ["PIP_SHIMS_BASE_MODULE"] = "pipenv.patched.notpip"
+os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip")
 
 PIPENV_CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv"))
 """Location for Pipenv to store it's package cache.

pipenv/patched/notpip/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file)
+Copyright (c) 2008-2018 The pip developers (see AUTHORS.txt file)
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

pipenv/patched/notpip/__init__.py

@@ -1 +1 @@
-__version__ = "10.0.1"
+__version__ = "18.0"

pipenv/patched/notpip/__main__.py

@@ -13,7 +13,7 @@
  path = os.path.dirname(os.path.dirname(__file__))
  sys.path.insert(0, path)
 
-from pipenv.patched.notpip._internal import main as _main # noqa
+from pipenv.patched.notpip._internal import main as _main # isort:skip # noqa
 
 if __name__ == '__main__':
  sys.exit(_main())