Skip to content

Commit

Permalink
Lock vcs deps to exact commit with dependencies
Browse files Browse the repository at this point in the history 
 - Manually obtain and update VCS repository with exact commit
 - Always store exact commit in the lockfile
 - Fixes #2180, #1690, #1611, #2096

Signed-off-by: Dan Ryan <dan@danryan.co>
  • Loading branch information
@techalchemy
techalchemy committed May 16, 2018
1 parent ea6f02d commit be6f4b8
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 85 deletions.
105 changes: 20 additions & 85 deletions pipenv/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -997,6 +997,7 @@ def do_lock(
):
"""Executes the freeze functionality."""
from notpip._vendor.distlib.markers import Evaluator
from .utils import get_vcs_deps
allowed_marker_keys = ['markers'] + [k for k in Evaluator.allowed_values.keys()]
cached_lockfile = {}
if not pre:
Expand Down Expand Up @@ -1033,6 +1034,9 @@ def do_lock(
if dev_package in project.packages:
dev_packages[dev_package] = project.packages[dev_package]
# Resolve dev-package dependencies, with pip-tools.
pip_freeze = delegator.run(
'{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system)))
).out
deps = convert_deps_to_pip(
dev_packages, project, r=False, include_index=True
)
Expand Down Expand Up @@ -1064,51 +1068,14 @@ def do_lock(
lockfile['develop'][dep['name']]['markers'] = dep['markers']
# Add refs for VCS installs.
# TODO: be smarter about this.
vcs_deps = convert_deps_to_pip(project.vcs_dev_packages, project, r=False)
pip_freeze = delegator.run(
'{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system)))
).out
vcs_registry = vcs()
if vcs_deps:
vcs_uri_map = {extract_uri_from_vcs_dep(v): k for k, v in project.vcs_dev_packages.items()}
for line in pip_freeze.strip().split('\n'):
# if the line doesn't match a vcs dependency in the Pipfile,
# ignore it
_vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line)
if not _vcs_match:
continue

pipfile_name = vcs_uri_map[_vcs_match]
src_loc = os.path.join(project.virtualenv_location, pipfile_name)
pipfile_rev = project.vcs_packages[pipfile_name].get('ref', None)
_pip_uri = line.lstrip('-e ')
backend_name = str(_pip_uri.split('+', 1)[0])
backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
__vcs = backend(url=_pip_uri)
__target_rev = __vcs.make_rev_options(pipfile_rev)

try:
installed = convert_deps_from_pip(line)
lock_name = first(installed.keys())
locked_rev = None
lockfile_src_loc = os.path.join(project.virtualenv_location, lock_name)
paths = set([loc for loc in [src_loc, lockfile_src_loc] if os.path.exists(loc)])
paths = list(paths)
if paths:
# If the pipfile rev and the installed rev don't match
if pipfile_rev != __vcs.get_url_rev()[1]:
for _p in paths:
__vcs.update(_p, __target_rev)
locked_rev = __vcs.get_revision(_p)
else:
__vcs.obtain(src_loc)
__vcs.update(src_loc, __target_rev)
locked_rev = __vcs.get_revision(src_loc)
if is_vcs(installed[lock_name]):
installed[lock_name]['ref'] = locked_rev
lockfile['develop'].update({pipfile_name: installed[lock_name]})
except IndexError:
pass
vcs_dev_lines, vcs_dev_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=True)
for lf in vcs_dev_lockfiles:
try:
name = first(lf.keys())
except AttributeError:
continue
if hasattr(lf[name], 'keys'):
lockfile['develop'].update(lf)
if write:
# Alert the user of progress.
click.echo(
Expand Down Expand Up @@ -1157,46 +1124,14 @@ def do_lock(
lockfile['default'][dep['name']]['markers'] = dep['markers']
# Add refs for VCS installs.
# TODO: be smarter about this.
vcs_deps = convert_deps_to_pip(project.vcs_packages, project, r=False)
if vcs_deps:
vcs_uri_map = {extract_uri_from_vcs_dep(v): k for k, v in project.vcs_packages.items()}
for line in pip_freeze.strip().split('\n'):
# if the line doesn't match a vcs dependency in the Pipfile,
# ignore it
_vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line)
if not _vcs_match:
continue

pipfile_name = vcs_uri_map[_vcs_match]
src_loc = os.path.join(project.virtualenv_location, pipfile_name)
pipfile_rev = project.vcs_packages[pipfile_name].get('ref', None)
_pip_uri = line.lstrip('-e ')
backend_name = str(_pip_uri.split('+', 1)[0])
backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
__vcs = backend(url=_pip_uri)
__target_rev = __vcs.make_rev_options(pipfile_rev)

try:
installed = convert_deps_from_pip(line)
lock_name = first(installed.keys())
locked_rev = None
lockfile_src_loc = os.path.join(project.virtualenv_location, lock_name)
paths = [loc for loc in [src_loc, lockfile_src_loc] if os.path.exists(loc)]
if paths:
# If the pipfile rev and the installed rev don't match
if pipfile_rev != __vcs.get_url_rev()[1]:
for _p in paths:
__vcs.update(_p, __target_rev)
locked_rev = __vcs.get_revision(_p)
else:
__vcs.obtain(src_loc)
__vcs.update(src_loc, __target_rev)
locked_rev = __vcs.get_revision(src_loc)
if is_vcs(installed[lock_name]):
installed[lock_name]['ref'] = locked_rev
lockfile['default'].update({pipfile_name: installed[lock_name]})
except IndexError:
pass
_vcs_deps, vcs_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=False)
for lf in vcs_lockfiles:
try:
name = first(lf.keys())
except AttributeError:
continue
if hasattr(lf[name], 'keys'):
lockfile['default'].update(lf)

# Support for --keep-outdated…
if keep_outdated:
Expand Down
61 changes: 61 additions & 0 deletions pipenv/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1365,3 +1365,64 @@ def extract_uri_from_vcs_dep(dep):
if hasattr(dep, 'keys'):
return first(dep[k] for k in valid_keys if k in dep) or None
return None


def install_or_update_vcs(vcs_obj, src_dir, name, rev=None):
target_dir = os.path.join(src_dir, name)
target_rev = vcs_obj.make_rev_options(rev)
if not os.path.exists(target_dir):
vcs_obj.obtain(target_dir)
vcs_obj.update(target_dir, target_rev)
return vcs_obj.get_revision(target_dir)


def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False):
from ._compat import vcs
section = 'vcs_dev_packages' if dev else 'vcs_packages'
lines = []
lockfiles = []
try:
packages = getattr(project, section)
except AttributeError:
return [], []
vcs_registry = vcs()
vcs_uri_map = {
extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')}
for k, v in packages.items()
}
for line in pip_freeze.strip().split('\n'):
# if the line doesn't match a vcs dependency in the Pipfile,
# ignore it
_vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line)
if not _vcs_match:
continue

pipfile_name = vcs_uri_map[_vcs_match]['name']
pipfile_rev = vcs_uri_map[_vcs_match]['ref']
src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src'))
mkdir_p(src_dir)
names = {pipfile_name}
_pip_uri = line.lstrip('-e ')
backend_name = str(_pip_uri.split('+', 1)[0])
backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
__vcs = backend(url=_pip_uri)

installed = convert_deps_from_pip(line)
if not hasattr(installed, 'keys'):
pass
lock_name = first(installed.keys())
names.add(lock_name)
locked_rev = None
for _name in names:
locked_rev = install_or_update_vcs(__vcs, src_dir, _name, rev=pipfile_rev)
if is_vcs(installed[lock_name]):
installed[lock_name]['ref'] = locked_rev
lockfiles.append({pipfile_name: installed[lock_name]})
pipfile_srcdir = os.path.join(src_dir, pipfile_name)
lockfile_srcdir = os.path.join(src_dir, lock_name)
lines.append(line)
if os.path.exists(pipfile_srcdir):
lockfiles.extend(venv_resolve_deps(['-e {0}'.format(pipfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global))
else:
lockfiles.extend(venv_resolve_deps(['-e {0}'.format(lockfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global))
return lines, lockfiles
26 changes: 26 additions & 0 deletions tests/integration/test_install_uri.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
import os
from flaky import flaky
import delegator
try:
from pathlib import Path
except ImportError:
from pathlib2 import Path


@pytest.mark.vcs
Expand Down Expand Up @@ -144,3 +148,25 @@ def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir):
assert six_key in p.lockfile['default']
# Make sure we didn't put six in the lockfile by accident as a vcs ref
assert 'six' not in p.lockfile['default']


@pytest.mark.vcs
@pytest.mark.install
@pytest.mark.needs_internet
def test_get_vcs_refs(PipenvInstance, pip_src_dir):
with PipenvInstance(chdir=True) as p:
c = p.pipenv('install -e git+https://github.com/hynek/structlog.git@16.1.0#egg=structlog')
assert c.return_code == 0
assert 'structlog' in p.pipfile['packages']
assert 'structlog' in p.lockfile['default']
assert 'six' in p.lockfile['default']
assert p.lockfile['default']['structlog']['ref'] == 'a39f6906a268fb2f4c365042b31d0200468fb492'
pipfile = Path(p.pipfile_path)
new_content = pipfile.read_bytes().replace(b'16.1.0', b'18.1.0')
pipfile.write_bytes(new_content)
c = p.pipenv('lock')
assert c.return_code == 0
assert p.lockfile['default']['structlog']['ref'] == 'a73fbd3a9c3cafb11f43168582083f839b883034'
assert 'structlog' in p.pipfile['packages']
assert 'structlog' in p.lockfile['default']
assert 'six' in p.lockfile['default']

0 comments on commit be6f4b8

Please sign in to comment.