From 15916afca676368ef9a8fa412fa1bc982db9e9ec Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 31 Aug 2018 01:53:55 -0400 Subject: [PATCH 01/21] Refactor CLI options - Allow parsing of multiple packages, editable and non-editable, interspersed - Handle `--index` and `--extra-index-url` with click's native parsing - Split commonly used options out into separate groups --- pipenv/cli.py | 515 +++++++++++++++++--------------------------------- 1 file changed, 176 insertions(+), 339 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 908c6f998c..8333e6b2ef 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -5,6 +5,7 @@ argument, command, echo, + secho, edit, group, Group, @@ -87,6 +88,109 @@ def validate_pypi_mirror(ctx, param, value): return value +def pypi_mirror(fn): + return option( + "--pypi-mirror", + default=environments.PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", + )(fn) + + +def python_group(fn): + fn = option( + "--three/--two", + is_flag=True, + default=None, + help="Use Python 3/2 when creating virtualenv.", + )(fn) + fn = option( + "--python", + default=False, + nargs=1, + callback=validate_python_path, + help="Specify which version of Python virtualenv should use.", + )(fn) + return fn + + +def package_group(fn): + fn = option( + "--editable", "-e", + help=u"An editable package to install.", + multiple=True + )(fn) + fn = argument( + "packages", + nargs=-1 + )(fn) + return fn + + +def common_group(fn): + fn = pypi_mirror(fn) + fn = option( + "--verbose", + "-v", + is_flag=True, + expose_value=False, + callback=setup_verbosity, + help="Verbose mode.", + )(fn) + return fn + + +def install_group(fn): + fn = option( + "--sequential", + is_flag=True, + default=False, + help="Install dependencies one-at-a-time, instead of concurrently.", + )(fn) + fn = option( + "--dev", + "-d", + is_flag=True, + default=False, + help="Install package(s) in [dev-packages].", + )(fn) + return fn + + +def upgrade_strategy_group(fn): + fn = option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.")(fn) + fn = option( + "--keep-outdated", + is_flag=True, + default=False, + help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", + )(fn) + fn = option( + "--selective-upgrade", + is_flag=True, + default=False, + help="Update specified packages.", + )(fn) + return fn + + +def index_group(fn): + fn = option( + "--index", "-i", + envvar="PIP_INDEX_URL", + nargs=1, + default=False, + help=u"Default PyPI compatible index URL to query for package lookups." + )(fn) + fn = option( + "--extra-index-url", + multiple=True, + help=u"URLs to the extra PyPI compatible indexes to query for package lookups." + )(fn) + return fn + + @group(cls=PipenvGroup, invoke_without_command=True, context_settings=CONTEXT_SETTINGS) @option("--where", is_flag=True, default=False, help="Output project home information.") @option("--venv", is_flag=True, default=False, help="Output virtualenv information.") @@ -105,32 +209,14 @@ def validate_pypi_mirror(ctx, param, value): help="Output completion (to be eval'd).", ) @option("--man", is_flag=True, default=False, help="Display manpage.") -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) +@python_group @option( "--site-packages", is_flag=True, default=False, help="Enable site-packages for the virtualenv.", ) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) +@pypi_mirror @option( "--support", is_flag=True, @@ -169,9 +255,9 @@ def cli( "variable.".format(crayons.normal("PIPENV_SHELL", bold=True)), err=True, ) - sys.exit(1) + ctx.abort() print(click_completion.get_code(shell=shell, prog_name="pipenv")) - sys.exit(0) + return 0 from .core import ( system_which, @@ -190,8 +276,10 @@ def cli( if system_which("man"): path = os.sep.join([os.path.dirname(__file__), "pipenv.1"]) os.execle(system_which("man"), "man", path, os.environ) + return 0 else: - echo("man does not appear to be available on your system.", err=True) + secho("man does not appear to be available on your system.", fg="yellow", bold=True, err=True) + return 1 if envs: echo("The following environment variables can be set, to do various things:\n") for key in environments.__dict__: @@ -204,26 +292,26 @@ def cli( ) ) ) - sys.exit(0) + return 0 warn_in_virtualenv() if ctx.invoked_subcommand is None: # --where was passed… if where: do_where(bare=True) - sys.exit(0) + return 0 elif py: do_py() - sys.exit() + return 0 # --support was passed… elif support: from .help import get_pipenv_diagnostics get_pipenv_diagnostics() - sys.exit(0) + return 0 # --clear was passed… elif clear: do_clear() - sys.exit(0) + return 0 # --venv was passed… elif venv: @@ -233,10 +321,10 @@ def cli( crayons.red("No virtualenv has been created for this project yet!"), err=True, ) - sys.exit(1) + ctx.abort() else: echo(project.virtualenv_location) - sys.exit(0) + return 0 # --rm was passed… elif rm: # Abort if --system (or running in a virtualenv). @@ -247,7 +335,7 @@ def cli( "Pipenv did not create. Aborting." ) ) - sys.exit(1) + ctx.abort() if project.virtualenv_exists: loc = project.virtualenv_location echo( @@ -261,7 +349,7 @@ def cli( with spinner(): # Remove the virtualenv. cleanup_virtualenv(bare=True) - sys.exit(0) + return 0 else: echo( crayons.red( @@ -270,7 +358,7 @@ def cli( ), err=True, ) - sys.exit(1) + ctx.abort() # --two / --three was passed… if (python or three is not None) or site_packages: ensure_project( @@ -291,35 +379,9 @@ def cli( short_help="Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.", context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), ) -@argument("package_name", default=False) -@argument("more_packages", nargs=-1) -@option( - "--dev", - "-d", - is_flag=True, - default=False, - help="Install package(s) in [dev-packages].", -) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) +@install_group +@python_group +@common_group @option("--system", is_flag=True, default=False, help="System pip management.") @option( "--requirements", @@ -330,12 +392,10 @@ def cli( ) @option("--code", "-c", nargs=1, default=False, help="Import from codebase.") @option( - "--verbose", - "-v", + "--skip-lock", is_flag=True, - expose_value=False, - callback=setup_verbosity, - help="Verbose mode.", + default=False, + help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", ) @option( "--ignore-pipfile", @@ -343,40 +403,16 @@ def cli( default=False, help="Ignore Pipfile when installing, using the Pipfile.lock.", ) -@option( - "--sequential", - is_flag=True, - default=False, - help="Install dependencies one-at-a-time, instead of concurrently.", -) -@option( - "--skip-lock", - is_flag=True, - default=False, - help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", -) @option( "--deploy", is_flag=True, default=False, help=u"Abort if the Pipfile.lock is out-of-date, or Python version is wrong.", ) -@option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.") -@option( - "--keep-outdated", - is_flag=True, - default=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", -) -@option( - "--selective-upgrade", - is_flag=True, - default=False, - help="Update specified packages.", -) +@upgrade_strategy_group +@index_group +@package_group def install( - package_name=False, - more_packages=False, dev=False, three=False, python=False, @@ -392,13 +428,15 @@ def install( deploy=False, keep_outdated=False, selective_upgrade=False, + index=False, + extra_index_url=False, + editable=False, + packages=False, ): """Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.""" from .core import do_install - do_install( - package_name=package_name, - more_packages=more_packages, + retcode = do_install( dev=dev, three=three, python=python, @@ -414,35 +452,20 @@ def install( deploy=deploy, keep_outdated=keep_outdated, selective_upgrade=selective_upgrade, + index_url=index, + extra_index_url=extra_index_url, + packages=packages, + editable_packages=editable, ) + if retcode: + ctx.abort() @command(short_help="Un-installs a provided package and removes it from Pipfile.") -@argument("package_name", default=False) -@argument("more_packages", nargs=-1) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) +@python_group @option("--system", is_flag=True, default=False, help="System pip management.") -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - help="Verbose mode.", - callback=setup_verbosity, -) @option("--lock", is_flag=True, default=True, help="Lock afterwards.") +@common_group @option( "--all-dev", is_flag=True, @@ -461,13 +484,7 @@ def install( default=False, help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", ) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) +@package_group def uninstall( package_name=False, more_packages=False, @@ -479,13 +496,15 @@ def uninstall( all=False, keep_outdated=False, pypi_mirror=None, + editable=False, + packages=False, ): """Un-installs a provided package and removes it from Pipfile.""" from .core import do_uninstall - do_uninstall( - package_name=package_name, - more_packages=more_packages, + retcode = do_uninstall( + packages=packages, + editable_packages=editable, three=three, python=python, system=system, @@ -495,37 +514,13 @@ def uninstall( keep_outdated=keep_outdated, pypi_mirror=pypi_mirror, ) + if retcode: + ctx.abort() @command(short_help="Generates Pipfile.lock.") -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - help="Verbose mode.", - callback=setup_verbosity, -) +@python_group +@common_group @option( "--requirements", "-r", @@ -577,19 +572,7 @@ def lock( short_help="Spawns a shell within the virtualenv.", context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), ) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) +@python_group @option( "--fancy", is_flag=True, @@ -602,13 +585,7 @@ def lock( default=False, help="Always spawn a subshell, even if one is already spawned.", ) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) +@pypi_mirror @argument("shell_args", nargs=-1) def shell( three=None, @@ -660,26 +637,8 @@ def shell( ) @argument("command") @argument("args", nargs=-1) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) +@python_group +@pypi_mirror def run(command, args, three=None, python=False, pypi_mirror=None): """Spawns a command installed into the virtualenv.""" from .core import do_run @@ -693,19 +652,7 @@ def run(command, args, three=None, python=False, pypi_mirror=None): short_help="Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.", context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), ) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) +@python_group @option("--system", is_flag=True, default=False, help="Use system Python.") @option( "--unused", @@ -719,13 +666,7 @@ def run(command, args, three=None, python=False, pypi_mirror=None): multiple=True, help="Ignore specified vulnerability during safety checks.", ) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) +@pypi_mirror @argument("args", nargs=-1) def check( three=None, @@ -753,41 +694,9 @@ def check( @command(short_help="Runs lock, then sync.") @argument("more_packages", nargs=-1) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - help="Verbose mode.", - callback=setup_verbosity, -) -@option( - "--dev", - "-d", - is_flag=True, - default=False, - help="Install package(s) in [dev-packages].", -) +@python_group +@common_group +@install_group @option("--clear", is_flag=True, default=False, help="Clear the dependency cache.") @option("--bare", is_flag=True, default=False, help="Minimal output.") @option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.") @@ -797,17 +706,11 @@ def check( default=False, help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", ) -@option( - "--sequential", - is_flag=True, - default=False, - help="Install dependencies one-at-a-time, instead of concurrently.", -) @option( "--outdated", is_flag=True, default=False, help=u"List out-of-date dependencies." ) @option("--dry-run", is_flag=True, default=None, help=u"List out-of-date dependencies.") -@argument("package", default=False) +@package_group @pass_context def update( ctx, @@ -821,10 +724,10 @@ def update( dev=False, bare=False, sequential=False, - package=None, dry_run=None, outdated=False, - more_packages=None, + packages=False, + editable=False, ): """Runs lock, then sync.""" from .core import ( @@ -840,7 +743,9 @@ def update( outdated = bool(dry_run) if outdated: do_outdated(pypi_mirror=pypi_mirror) - if not package: + packages = [p for p in packages if p] + editable = [p for p in editable if p] + if not packages: echo( "{0} {1} {2} {3}{4}".format( crayons.white("Running", bold=True), @@ -851,7 +756,7 @@ def update( ) ) else: - for package in [package] + list(more_packages) or []: + for package in packages + editable: if package not in project.all_packages: echo( "{0}: {1} was not found in your Pipfile! Aborting." @@ -861,7 +766,7 @@ def update( ), err=True, ) - sys.exit(1) + ctx.abort do_lock( clear=clear, @@ -897,26 +802,8 @@ def graph(bare=False, json=False, json_tree=False, reverse=False): @command(short_help="View a given module in your editor.", name="open") -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) +@python_group +@pypi_mirror @argument("module", nargs=1) def run_open(module, three=None, python=None, pypi_mirror=None): """View a given module in your editor.""" @@ -942,49 +829,11 @@ def run_open(module, three=None, python=None, pypi_mirror=None): @command(short_help="Installs all packages specified in Pipfile.lock.") -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - help="Verbose mode.", - callback=setup_verbosity, -) -@option( - "--dev", - "-d", - is_flag=True, - default=False, - help="Additionally install package(s) in [dev-packages].", -) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) +@common_group +@install_group +@python_group @option("--bare", is_flag=True, default=False, help="Minimal output.") @option("--clear", is_flag=True, default=False, help="Clear the dependency cache.") -@option( - "--sequential", - is_flag=True, - default=False, - help="Install dependencies one-at-a-time, instead of concurrently.", -) @pass_context def sync( ctx, @@ -1027,19 +876,7 @@ def sync( help="Verbose mode.", callback=setup_verbosity, ) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) +@python_group @option( "--dry-run", is_flag=True, From eb18ed260f0ecbaf1493d2862206058b00306070 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 31 Aug 2018 01:56:32 -0400 Subject: [PATCH 02/21] Clean up core arguments - Handle packages and editable packages separately - Allow `pip_install` to take `Requirement` objects as arguments rather than re-parsing - Remove bad parsing logic --- pipenv/core.py | 181 ++++++++++++++++++++++--------------------------- 1 file changed, 82 insertions(+), 99 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index bab12f937d..a338063f0c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -655,7 +655,7 @@ def do_install_dependencies( If requirements is True, simply spits out a requirements format to stdout. """ - + from .vendor.requirementslib import Requirement def cleanup_procs(procs, concurrent): for c in procs: if concurrent: @@ -739,6 +739,7 @@ def cleanup_procs(procs, concurrent): # Use a specific index, if specified. dep, index = split_argument(dep, short="i", long_="index", num=1) dep, extra_indexes = split_argument(dep, long_="extra-index-url") + dep = Requirement.from_line(dep) # Install the module. c = pip_install( dep, @@ -767,6 +768,7 @@ def cleanup_procs(procs, concurrent): # Use a specific index, if specified. dep, index = split_argument(dep, short="i", long_="index", num=1) dep, extra_indexes = split_argument(dep, long_="extra-index-url") + dep = Requirement.from_line(dep) # Install the module. c = pip_install( dep, @@ -788,7 +790,7 @@ def cleanup_procs(procs, concurrent): click.echo( "{0} {1}{2}".format( crayons.green("Success installing"), - crayons.green(dep.split("--hash")[0].strip()), + crayons.green(dep.name), crayons.green("!"), ) ) @@ -1255,7 +1257,7 @@ def do_init( def pip_install( - package_name=None, + requirement=None, r=None, allow_global=False, ignore_hashes=False, @@ -1273,37 +1275,21 @@ def pip_install( from .vendor.requirementslib import Requirement if environments.is_verbose(): - click.echo( - crayons.normal("Installing {0!r}".format(package_name), bold=True), err=True - ) piplogger.setLevel(logging.INFO) + if requirement: + click.echo( + crayons.normal("Installing {0!r}".format(requirement.name), bold=True), + err=True + ) # Create files for hash mode. - if not package_name.startswith("-e ") and (not ignore_hashes) and (r is None): + if requirement and not requirement.editable and (not ignore_hashes) and (r is None): fd, r = tempfile.mkstemp( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir ) with os.fdopen(fd, "w") as f: - f.write(package_name) + f.write(requirement.normalized_name) # Install dependencies when a package is a VCS dependency. - try: - req = Requirement.from_line( - package_name.split("--hash")[0].split("--trusted-host")[0] - ).vcs - except (ParseException, ValueError) as e: - click.echo("{0}: {1}".format(crayons.red("WARNING"), e), err=True) - click.echo( - "{0}… You will have to reinstall any packages that failed to install.".format( - crayons.red("ABORTING INSTALL") - ), - err=True, - ) - click.echo( - "You may have to manually run {0} when you are finished.".format( - crayons.normal("pipenv lock", bold=True) - ) - ) - sys.exit(1) - if req: + if requirement and requirement.vcs: no_deps = False # Don't specify a source directory when using --system. if not allow_global and ("PIP_SRC" not in os.environ): @@ -1341,12 +1327,12 @@ def pip_install( create_mirror_source(pypi_mirror) if is_pypi_url(source["url"]) else source for source in sources ] - if package_name.startswith("-e "): - install_reqs = ' -e "{0}"'.format(package_name.split("-e ")[1]) + if requirement and requirement.editable: + install_reqs = ' {0}'.format(requirement.as_line()) elif r: install_reqs = " -r {0}".format(escape_grouped_arguments(r)) else: - install_reqs = ' "{0}"'.format(package_name) + install_reqs = ' "{0}"'.format(requirement.as_line()) # Skip hash-checking mode, when appropriate. if r: with open(r) as f: @@ -1623,8 +1609,10 @@ def do_outdated(pypi_mirror=None): def do_install( - package_name=False, - more_packages=False, + packages=False, + editable_packages=False, + index_url=False, + extra_index_url=False, dev=False, three=False, python=False, @@ -1649,12 +1637,13 @@ def do_install( ) if selective_upgrade: keep_outdated = True - more_packages = more_packages or [] + packages = packages if packages else [] + editable_packages = editable_packages if editable_packages else [] + package_args = [p for p in packages if p] + [p for p in editable_packages if p] + skip_requirements = False # Don't search for requirements.txt files if the user provides one - if requirements or package_name or project.pipfile_exists: + if requirements or package_args or project.pipfile_exists: skip_requirements = True - else: - skip_requirements = False concurrent = not sequential # Ensure that virtualenv is available. ensure_project( @@ -1673,7 +1662,7 @@ def do_install( keep_outdated = project.settings.get("keep_outdated") remote = requirements and is_valid_url(requirements) # Warn and exit if --system is used without a pipfile. - if (system and package_name) and not (PIPENV_VIRTUALENV): + if (system and package_args) and not (PIPENV_VIRTUALENV): click.echo( "{0}: --system is intended to be used for Pipfile installation, " "not installation of specific packages. Aborting.".format( @@ -1757,32 +1746,9 @@ def do_install( for req in import_from_code(code): click.echo(" Found {0}!".format(crayons.green(req))) project.add_package_to_pipfile(req) - # Capture -e argument and assign it to following package_name. - more_packages = list(more_packages) - if package_name == "-e": - if not more_packages: - raise click.BadArgumentUsage("Please provide path to editable package") - package_name = " ".join([package_name, more_packages.pop(0)]) - # capture indexes and extra indexes - line = [package_name] + more_packages - line = " ".join(str(s) for s in line).strip() - index_indicators = ["-i", "--index", "--extra-index-url"] - index, extra_indexes = None, None - if any(line.endswith(s) for s in index_indicators): - # check if cli option is not end of command - raise click.BadArgumentUsage("Please provide index value") - if any(s in line for s in index_indicators): - line, index = split_argument(line, short="i", long_="index", num=1) - line, extra_indexes = split_argument(line, long_="extra-index-url") - package_names = line.split() - package_name = package_names[0] - if len(package_names) > 1: - more_packages = package_names[1:] - else: - more_packages = [] # Capture . argument and assign it to nothing - if package_name == ".": - package_name = False + if len(packages) == 1 and packages[0] == ".": + packages = False # Install editable local packages before locking - this gives us access to dist-info if project.pipfile_exists and ( # double negatives are for english readability, leave them alone. @@ -1793,38 +1759,40 @@ def do_install( project.editable_packages if not dev else project.editable_dev_packages ) for package in section.keys(): - converted = convert_deps_to_pip( + req = convert_deps_to_pip( {package: section[package]}, project=project, r=False ) - if not package_name: - if converted: - package_name = converted.pop(0) - if converted: - more_packages.extend(converted) + if req: + req = req[0] + req = req[len("-e "):] if req.startswith("-e ") else req + if not editable_packages: + editable_packages = [req] + else: + editable_packages.extend([req]) # Allow more than one package to be provided. - package_names = [package_name] + more_packages + package_args = [p for p in packages] + ["-e {0}".format(pkg) for pkg in editable_packages] # Support for --selective-upgrade. # We should do this part first to make sure that we actually do selectively upgrade # the items specified if selective_upgrade: from .vendor.requirementslib import Requirement - for i, package_name in enumerate(package_names[:]): + for i, package in enumerate(package_args[:]): section = project.packages if not dev else project.dev_packages - package = Requirement.from_line(package_name) + package = Requirement.from_line(package) package__name, package__val = package.pipfile_entry try: if not is_star(section[package__name]) and is_star(package__val): # Support for VCS dependencies. - package_names[i] = convert_deps_to_pip( - {package_name: section[package__name]}, project=project, r=False + package_args[i] = convert_deps_to_pip( + {packages: section[package__name]}, project=project, r=False )[0] except KeyError: pass # Install all dependencies, if none was provided. # This basically ensures that we have a pipfile and lockfile, then it locks and # installs from the lockfile - if package_name is False: + if packages is False and editable_packages is False: # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) @@ -1844,36 +1812,45 @@ def do_install( # This is for if the user passed in dependencies, then we want to maek sure we else: from .vendor.requirementslib import Requirement - - for package_name in package_names: + # make a tuple of (display_name, entry) + pkg_dict = { + 'packages': [(pkg, pkg) for pkg in packages], + 'editables': [("-e {0}".format(pkg), pkg) for pkg in editable_packages] + } + + for pkg_type, pkg_tuple in pkg_dict.items(): + if not pkg_tuple: + continue + pkg_line, pkg_val = pkg_tuple.pop() click.echo( crayons.normal( - u"Installing {0}…".format(crayons.green(package_name, bold=True)), + u"Installing {0}…".format(crayons.green(pkg_line, bold=True)), bold=True, ) ) # pip install: with spinner(): + try: + pkg_requirement = Requirement.from_line(pkg_line) + except ValueError as e: + click.echo("{0}: {1}".format(crayons.red("WARNING"), e)) + requirements_directory.cleanup() + sys.exit(1) + c = pip_install( - package_name, + pkg_requirement, ignore_hashes=True, allow_global=system, selective_upgrade=selective_upgrade, no_deps=False, pre=pre, requirements_dir=requirements_directory.name, - index=index, - extra_indexes=extra_indexes, + index=index_url, + extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) # Warn if --editable wasn't passed. - try: - converted = Requirement.from_line(package_name) - except ValueError as e: - click.echo("{0}: {1}".format(crayons.red("WARNING"), e)) - requirements_directory.cleanup() - sys.exit(1) - if converted.is_vcs and not converted.editable: + if pkg_requirement.is_vcs and not pkg_requirement.editable: click.echo( "{0}: You installed a VCS dependency in non-editable mode. " "This will work fine, but sub-dependencies will not be resolved by {1}." @@ -1890,7 +1867,7 @@ def do_install( except AssertionError: click.echo( "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(package_name) + crayons.red("Error: ", bold=True), crayons.green(pkg_line) ), err=True, ) @@ -1899,7 +1876,7 @@ def do_install( click.echo( "This is likely caused by a bug in {0}. " "Report this to its maintainers.".format( - crayons.green(package_name) + crayons.green(pkg_requirement.name) ), err=True, ) @@ -1908,7 +1885,7 @@ def do_install( click.echo( "{0} {1} {2} {3}{4}".format( crayons.normal("Adding", bold=True), - crayons.green(package_name, bold=True), + crayons.green(pkg_requirement.name, bold=True), crayons.normal("to Pipfile's", bold=True), crayons.red("[dev-packages]" if dev else "[packages]", bold=True), crayons.normal("…", bold=True), @@ -1916,7 +1893,7 @@ def do_install( ) # Add the package to the Pipfile. try: - project.add_package_to_pipfile(package_name, dev) + project.add_package_to_pipfile(pkg_requirement.as_line(), dev) except ValueError as e: click.echo( "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), e) @@ -1940,8 +1917,8 @@ def do_install( def do_uninstall( - package_name=False, - more_packages=False, + packages=False, + editable_packages=False, three=None, python=False, system=False, @@ -1952,13 +1929,19 @@ def do_uninstall( pypi_mirror=None, ): from .environments import PIPENV_USE_SYSTEM + from .vendor.requirementslib import Requirement # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True # Ensure that virtualenv is available. ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) - package_names = (package_name,) + more_packages + editable_pkgs = [ + Requirement.from_line("-e {0}".format(p)).name + for p in editable_packages + if p + ] + package_names = [p for p in packages if p] + editable_pkgs pipfile_remove = True # Un-install all dependencies, if --all was provided. if all is True: @@ -1966,7 +1949,7 @@ def do_uninstall( crayons.normal(u"Un-installing all packages from virtualenv…", bold=True) ) do_purge(allow_global=system) - sys.exit(0) + return # Uninstall [dev-packages], if --dev was provided. if all_dev: if "dev-packages" not in project.parsed_pipfile: @@ -1976,16 +1959,16 @@ def do_uninstall( bold=True, ) ) - sys.exit(0) + return click.echo( crayons.normal( u"Un-installing {0}…".format(crayons.red("[dev-packages]")), bold=True ) ) package_names = project.dev_packages.keys() - if package_name is False and not all_dev: + if packages is False and editable_packages is False and not all_dev: click.echo(crayons.red("No package provided!"), err=True) - sys.exit(1) + return 1 for package_name in package_names: click.echo(u"Un-installing {0}…".format(crayons.green(package_name))) cmd = "{0} uninstall {1} -y".format( @@ -2444,7 +2427,7 @@ def do_sync( ), err=True, ) - sys.exit(1) + return 1 # Ensure that virtualenv is available if not system. ensure_project( From 228b28b41d6bb150a2a039e5d7cc8f20f540ee87 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 31 Aug 2018 01:57:30 -0400 Subject: [PATCH 03/21] Update piptools patch - Handle editable installs more effectively - Actually store the distribution after we create it - Actually get the dependencies from it --- pipenv/patched/piptools/repositories/pypi.py | 11 +++++----- .../vendoring/patches/patched/piptools.patch | 21 +++++++++---------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index e3156f6a6b..699db8e30d 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -283,21 +283,20 @@ def get_legacy_dependencies(self, ireq): from pipenv.utils import chdir with chdir(ireq.setup_py_dir): from setuptools.dist import distutils - distutils.core.run_setup(ireq.setup_py) + dist = distutils.core.run_setup(ireq.setup_py) except (ImportError, InstallationError, TypeError, AttributeError): pass try: - dist = ireq.get_dist() + dist = ireq.get_dist() if not dist else dist except InstallationError: ireq.run_egg_info() dist = ireq.get_dist() except (TypeError, ValueError, AttributeError): pass else: - if dist.has_metadata('requires.txt'): - setup_requires = self.finder.get_extras_links( - dist.get_metadata_lines('requires.txt') - ) + setup_requires = getattr(dist, "requires", None) + if not setup_requires: + setup_requires = getattr(dist, "setup_requires", None) try: # Pip 9 and below reqset = RequirementSet( diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 7e8a14422d..6ae7011fb7 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..91902dc 100644 +index 1c4b943..10447b6 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,9 +1,10 @@ @@ -230,7 +230,7 @@ index 1c4b943..91902dc 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +273,47 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +273,46 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -245,21 +245,20 @@ index 1c4b943..91902dc 100644 + from pipenv.utils import chdir + with chdir(ireq.setup_py_dir): + from setuptools.dist import distutils -+ distutils.core.run_setup(ireq.setup_py) ++ dist = distutils.core.run_setup(ireq.setup_py) + except (ImportError, InstallationError, TypeError, AttributeError): + pass + try: -+ dist = ireq.get_dist() ++ dist = ireq.get_dist() if not dist else dist + except InstallationError: + ireq.run_egg_info() + dist = ireq.get_dist() + except (TypeError, ValueError, AttributeError): + pass + else: -+ if dist.has_metadata('requires.txt'): -+ setup_requires = self.finder.get_extras_links( -+ dist.get_metadata_lines('requires.txt') -+ ) ++ setup_requires = getattr(dist, "requires", None) ++ if not setup_requires: ++ setup_requires = getattr(dist, "setup_requires", None) try: - # Pip < 9 and below + # Pip 9 and below @@ -282,7 +281,7 @@ index 1c4b943..91902dc 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +333,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +332,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -383,7 +382,7 @@ index 1c4b943..91902dc 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -210,6 +435,10 @@ class PyPIRepository(BaseRepository): +@@ -210,6 +434,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -394,7 +393,7 @@ index 1c4b943..91902dc 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -217,24 +446,22 @@ class PyPIRepository(BaseRepository): +@@ -217,24 +445,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. From 7d5738673319ddd328ad675c95d232e965499904 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 31 Aug 2018 01:57:44 -0400 Subject: [PATCH 04/21] Clean up test --- tests/integration/test_install_twists.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index c9dded4314..055c39b2e1 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -13,15 +13,10 @@ @pytest.mark.extras @pytest.mark.install @pytest.mark.local -@pytest.mark.parametrize( - "line, pipfile", - [["-e .[dev]", {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}}]], -) -def test_local_extras_install(PipenvInstance, pypi, line, pipfile): +def test_local_extras_install(PipenvInstance, pypi): """Ensure -e .[extras] installs. """ with PipenvInstance(pypi=pypi, chdir=True) as p: - project = Project() setup_py = os.path.join(p.path, "setup.py") with open(setup_py, "w") as fh: contents = """ @@ -40,7 +35,17 @@ def test_local_extras_install(PipenvInstance, pypi, line, pipfile): ) """.strip() fh.write(contents) - project.write_toml({"packages": pipfile, "dev-packages": {}}) + line = "-e .[dev]" + # pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} + project = Project() + with open(os.path.join(p.path, 'Pipfile'), 'w') as fh: + fh.write(""" +[packages] +testpipenv = {path = ".", editable = true, extras = ["dev"]} + +[dev-packages] + """.strip()) + # project.write_toml({"packages": pipfile, "dev-packages": {}}) c = p.pipenv("install") assert c.return_code == 0 assert "testpipenv" in p.lockfile["default"] From 8b32e1979b8db2552e4fb15af6e578aee7593427 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 2 Sep 2018 01:18:05 -0400 Subject: [PATCH 05/21] Refactor CLI for organization and simplicity Signed-off-by: Dan Ryan --- pipenv/cli/__init__.py | 3 + pipenv/{cli.py => cli/command.py} | 630 +++++++----------------- pipenv/cli/options.py | 368 ++++++++++++++ pipenv/core.py | 9 +- tests/integration/test_install_basic.py | 2 +- tests/integration/test_sync.py | 4 +- 6 files changed, 546 insertions(+), 470 deletions(-) create mode 100644 pipenv/cli/__init__.py rename pipenv/{cli.py => cli/command.py} (50%) create mode 100644 pipenv/cli/options.py diff --git a/pipenv/cli/__init__.py b/pipenv/cli/__init__.py new file mode 100644 index 0000000000..1064ee3b14 --- /dev/null +++ b/pipenv/cli/__init__.py @@ -0,0 +1,3 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, unicode_literals +from .command import cli diff --git a/pipenv/cli.py b/pipenv/cli/command.py similarity index 50% rename from pipenv/cli.py rename to pipenv/cli/command.py index 8333e6b2ef..d7815d5f52 100644 --- a/pipenv/cli.py +++ b/pipenv/cli/command.py @@ -1,194 +1,32 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import, unicode_literals + import os import sys + +import crayons +import delegator + from click import ( - argument, - command, - echo, - secho, - edit, - group, - Group, - option, - pass_context, - Option, - version_option, - BadParameter, + argument, echo, edit, group, option, pass_context, secho, version_option ) -from click_didyoumean import DYMCommandCollection import click_completion -import crayons -import delegator -from .__version__ import __version__ +from click_didyoumean import DYMCommandCollection + +from .. import environments +from ..__version__ import __version__ +from .options import ( + CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, + general_options, install_options, lock_options, pass_state, + pypi_mirror_option, python_option, requirementstxt_option, sync_options, + system_option, three_option, verbose_option, uninstall_options +) -from . import environments -from .utils import is_valid_url # Enable shell completion. click_completion.init() -CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) - - -class PipenvGroup(Group): - """Custom Group class provides formatted main help""" - - def get_help_option(self, ctx): - from .core import format_help - - """Override for showing formatted main help via --help and -h options""" - help_options = self.get_help_option_names(ctx) - if not help_options or not self.add_help_option: - return - - def show_help(ctx, param, value): - if value and not ctx.resilient_parsing: - if not ctx.invoked_subcommand: - # legit main help - echo(format_help(ctx.get_help())) - else: - # legit sub-command help - echo(ctx.get_help(), color=ctx.color) - ctx.exit() - - return Option( - help_options, - is_flag=True, - is_eager=True, - expose_value=False, - callback=show_help, - help="Show this message and exit.", - ) - - -def setup_verbosity(ctx, param, value): - if not value: - return - import logging - logging.getLogger("pip").setLevel(logging.INFO) - environments.PIPENV_VERBOSITY = 1 - - -def validate_python_path(ctx, param, value): - # Validating the Python path is complicated by accepting a number of - # friendly options: the default will be boolean False to enable - # autodetection but it may also be a value which will be searched in - # the path or an absolute path. To report errors as early as possible - # we'll report absolute paths which do not exist: - if isinstance(value, (str, bytes)): - if os.path.isabs(value) and not os.path.isfile(value): - raise BadParameter("Expected Python at path %s does not exist" % value) - return value - - -def validate_pypi_mirror(ctx, param, value): - if value and not is_valid_url(value): - raise BadParameter("Invalid PyPI mirror URL: %s" % value) - return value - - -def pypi_mirror(fn): - return option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", - )(fn) - - -def python_group(fn): - fn = option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", - )(fn) - fn = option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", - )(fn) - return fn - - -def package_group(fn): - fn = option( - "--editable", "-e", - help=u"An editable package to install.", - multiple=True - )(fn) - fn = argument( - "packages", - nargs=-1 - )(fn) - return fn - - -def common_group(fn): - fn = pypi_mirror(fn) - fn = option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - callback=setup_verbosity, - help="Verbose mode.", - )(fn) - return fn - - -def install_group(fn): - fn = option( - "--sequential", - is_flag=True, - default=False, - help="Install dependencies one-at-a-time, instead of concurrently.", - )(fn) - fn = option( - "--dev", - "-d", - is_flag=True, - default=False, - help="Install package(s) in [dev-packages].", - )(fn) - return fn - - -def upgrade_strategy_group(fn): - fn = option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.")(fn) - fn = option( - "--keep-outdated", - is_flag=True, - default=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", - )(fn) - fn = option( - "--selective-upgrade", - is_flag=True, - default=False, - help="Update specified packages.", - )(fn) - return fn - - -def index_group(fn): - fn = option( - "--index", "-i", - envvar="PIP_INDEX_URL", - nargs=1, - default=False, - help=u"Default PyPI compatible index URL to query for package lookups." - )(fn) - fn = option( - "--extra-index-url", - multiple=True, - help=u"URLs to the extra PyPI compatible indexes to query for package lookups." - )(fn) - return fn @group(cls=PipenvGroup, invoke_without_command=True, context_settings=CONTEXT_SETTINGS) @@ -209,24 +47,18 @@ def index_group(fn): help="Output completion (to be eval'd).", ) @option("--man", is_flag=True, default=False, help="Display manpage.") -@python_group -@option( - "--site-packages", - is_flag=True, - default=False, - help="Enable site-packages for the virtualenv.", -) -@pypi_mirror @option( "--support", is_flag=True, help="Output diagnostic information for use in GitHub issues.", ) -@option("--clear", is_flag=True, help="Clears caches (pipenv, pip, and pip-tools).") +@general_options @version_option(prog_name=crayons.normal("pipenv", bold=True), version=__version__) +@pass_state @pass_context def cli( ctx, + state, where=False, venv=False, rm=False, @@ -242,10 +74,11 @@ def cli( pypi_mirror=None, support=None, clear=False, + **kwargs ): # Handle this ASAP to make shell startup fast. if completion: - from . import shells + from .. import shells try: shell = shells.detect_info()[0] @@ -259,7 +92,7 @@ def cli( print(click_completion.get_code(shell=shell, prog_name="pipenv")) return 0 - from .core import ( + from ..core import ( system_which, do_py, warn_in_virtualenv, @@ -304,7 +137,7 @@ def cli( return 0 # --support was passed… elif support: - from .help import get_pipenv_diagnostics + from ..help import get_pipenv_diagnostics get_pipenv_diagnostics() return 0 @@ -362,12 +195,12 @@ def cli( # --two / --three was passed… if (python or three is not None) or site_packages: ensure_project( - three=three, - python=python, + three=state.three, + python=state.python, warn=True, - site_packages=site_packages, - pypi_mirror=pypi_mirror, - clear=clear, + site_packages=state.site_packages, + pypi_mirror=state.pypi_mirror, + clear=state.clear, ) # Check this again before exiting for empty ``pipenv`` command. elif ctx.invoked_subcommand is None: @@ -375,97 +208,52 @@ def cli( echo(format_help(ctx.get_help())) -@command( +@cli.command( short_help="Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.", context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), ) -@install_group -@python_group -@common_group -@option("--system", is_flag=True, default=False, help="System pip management.") -@option( - "--requirements", - "-r", - nargs=1, - default=False, - help="Import a requirements.txt file.", -) -@option("--code", "-c", nargs=1, default=False, help="Import from codebase.") -@option( - "--skip-lock", - is_flag=True, - default=False, - help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", -) -@option( - "--ignore-pipfile", - is_flag=True, - default=False, - help="Ignore Pipfile when installing, using the Pipfile.lock.", -) -@option( - "--deploy", - is_flag=True, - default=False, - help=u"Abort if the Pipfile.lock is out-of-date, or Python version is wrong.", -) -@upgrade_strategy_group -@index_group -@package_group +@requirementstxt_option +@system_option +@code_option +@deploy_option +@install_options +@pass_state +@pass_context def install( - dev=False, - three=False, - python=False, - pypi_mirror=None, - system=False, - lock=True, - ignore_pipfile=False, - skip_lock=False, - requirements=False, - sequential=False, - pre=False, - code=False, - deploy=False, - keep_outdated=False, - selective_upgrade=False, - index=False, - extra_index_url=False, - editable=False, - packages=False, + ctx, + state, + **kwargs ): """Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.""" - from .core import do_install + from ..core import do_install retcode = do_install( - dev=dev, - three=three, - python=python, - pypi_mirror=pypi_mirror, - system=system, - lock=lock, - ignore_pipfile=ignore_pipfile, - skip_lock=skip_lock, - requirements=requirements, - sequential=sequential, - pre=pre, - code=code, - deploy=deploy, - keep_outdated=keep_outdated, - selective_upgrade=selective_upgrade, - index_url=index, - extra_index_url=extra_index_url, - packages=packages, - editable_packages=editable, + dev=state.installstate.dev, + three=state.three, + python=state.python, + pypi_mirror=state.pypi_mirror, + system=state.system, + lock=not state.installstate.skip_lock, + ignore_pipfile=state.installstate.ignore_pipfile, + skip_lock=state.installstate.skip_lock, + requirements=state.installstate.requirementstxt, + sequential=state.installstate.sequential, + pre=state.installstate.pre, + code=state.installstate.code, + deploy=state.installstate.deploy, + keep_outdated=state.installstate.keep_outdated, + selective_upgrade=state.installstate.selective_upgrade, + index_url=state.index, + extra_index_url=state.extra_index_urls, + packages=state.installstate.packages, + editable_packages=state.installstate.editables, ) if retcode: ctx.abort() -@command(short_help="Un-installs a provided package and removes it from Pipfile.") -@python_group -@option("--system", is_flag=True, default=False, help="System pip management.") +@cli.command(short_help="Un-installs a provided package and removes it from Pipfile.") @option("--lock", is_flag=True, default=True, help="Lock afterwards.") -@common_group @option( "--all-dev", is_flag=True, @@ -478,49 +266,35 @@ def install( default=False, help="Purge all package(s) from virtualenv. Does not edit Pipfile.", ) -@option( - "--keep-outdated", - is_flag=True, - default=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", -) -@package_group +@uninstall_options +@pass_state def uninstall( - package_name=False, - more_packages=False, - three=None, - python=False, - system=False, + state, lock=False, all_dev=False, all=False, - keep_outdated=False, - pypi_mirror=None, - editable=False, - packages=False, + **kwargs ): """Un-installs a provided package and removes it from Pipfile.""" - from .core import do_uninstall + from ..core import do_uninstall retcode = do_uninstall( - packages=packages, - editable_packages=editable, - three=three, - python=python, - system=system, + packages=state.installstate.packages, + editable_packages=state.installstate.editables, + three=state.three, + python=state.python, + system=state.system, lock=lock, all_dev=all_dev, all=all, - keep_outdated=keep_outdated, - pypi_mirror=pypi_mirror, + keep_outdated=state.installstate.keep_outdated, + pypi_mirror=state.pypi_mirror, ) if retcode: - ctx.abort() + sys.exit(retcode) -@command(short_help="Generates Pipfile.lock.") -@python_group -@common_group +@cli.command(short_help="Generates Pipfile.lock.") @option( "--requirements", "-r", @@ -528,51 +302,33 @@ def uninstall( default=False, help="Generate output compatible with requirements.txt.", ) -@option( - "--dev", - "-d", - is_flag=True, - default=False, - help="Generate output compatible with requirements.txt for the development dependencies.", -) -@option("--clear", is_flag=True, default=False, help="Clear the dependency cache.") -@option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.") -@option( - "--keep-outdated", - is_flag=True, - default=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", -) +@lock_options +@pass_state def lock( - three=None, - python=False, - pypi_mirror=None, + state, requirements=False, - dev=False, - clear=False, - pre=False, - keep_outdated=False, + **kwargs ): """Generates Pipfile.lock.""" - from .core import ensure_project, do_init, do_lock + from ..core import ensure_project, do_init, do_lock # Ensure that virtualenv is available. - ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) + ensure_project(three=state.three, python=state.python, pypi_mirror=state.pypi_mirror) if requirements: - do_init(dev=dev, requirements=requirements, pypi_mirror=pypi_mirror) + do_init(dev=state.installstate.dev, requirements=requirements, + pypi_mirror=state.pypi_mirror) do_lock( - clear=clear, - pre=pre, - keep_outdated=keep_outdated, - pypi_mirror=pypi_mirror, + clear=state.clear, + pre=state.pre, + keep_outdated=state.installstate.keep_outdated, + pypi_mirror=state.pypi_mirror, ) -@command( +@cli.command( short_help="Spawns a shell within the virtualenv.", context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), ) -@python_group @option( "--fancy", is_flag=True, @@ -585,9 +341,13 @@ def lock( default=False, help="Always spawn a subshell, even if one is already spawned.", ) -@pypi_mirror @argument("shell_args", nargs=-1) +@pypi_mirror_option +@three_option +@python_option +@pass_state def shell( + state, three=None, python=False, fancy=False, @@ -596,7 +356,7 @@ def shell( pypi_mirror=None, ): """Spawns a shell within the virtualenv.""" - from .core import load_dot_env, do_shell + from ..core import load_dot_env, do_shell # Prevent user from activating nested environments. if "PIPENV_ACTIVE" in os.environ: @@ -626,7 +386,7 @@ def shell( ) -@command( +@cli.command( add_help_option=False, short_help="Spawns a command installed into the virtualenv.", context_settings=dict( @@ -635,25 +395,23 @@ def shell( allow_extra_args=True, ), ) +@common_options @argument("command") @argument("args", nargs=-1) -@python_group -@pypi_mirror -def run(command, args, three=None, python=False, pypi_mirror=None): +@pass_state +def run(state, command, args): """Spawns a command installed into the virtualenv.""" - from .core import do_run + from ..core import do_run do_run( - command=command, args=args, three=three, python=python, pypi_mirror=pypi_mirror + command=command, args=args, three=state.three, python=state.python, pypi_mirror=state.pypi_mirror ) -@command( +@cli.command( short_help="Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.", context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), ) -@python_group -@option("--system", is_flag=True, default=False, help="Use system Python.") @option( "--unused", nargs=1, @@ -666,71 +424,51 @@ def run(command, args, three=None, python=False, pypi_mirror=None): multiple=True, help="Ignore specified vulnerability during safety checks.", ) -@pypi_mirror +@common_options +@system_option @argument("args", nargs=-1) +@pass_state def check( - three=None, - python=False, - system=False, + state, unused=False, style=False, ignore=None, args=None, - pypi_mirror=None, + **kwargs ): """Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.""" - from .core import do_check + from ..core import do_check do_check( - three=three, - python=python, - system=system, + three=state.three, + python=state.python, + system=state.system, unused=unused, ignore=ignore, args=args, - pypi_mirror=pypi_mirror, + pypi_mirror=state.pypi_mirror, ) -@command(short_help="Runs lock, then sync.") -@argument("more_packages", nargs=-1) -@python_group -@common_group -@install_group -@option("--clear", is_flag=True, default=False, help="Clear the dependency cache.") +@cli.command(short_help="Runs lock, then sync.") @option("--bare", is_flag=True, default=False, help="Minimal output.") -@option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.") -@option( - "--keep-outdated", - is_flag=True, - default=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", -) @option( "--outdated", is_flag=True, default=False, help=u"List out-of-date dependencies." ) @option("--dry-run", is_flag=True, default=None, help=u"List out-of-date dependencies.") -@package_group +@install_options +@pass_state @pass_context def update( ctx, - three=None, - python=False, - pypi_mirror=None, - system=False, - clear=False, - keep_outdated=False, - pre=False, - dev=False, + state, bare=False, - sequential=False, dry_run=None, outdated=False, - packages=False, - editable=False, + **kwargs ): """Runs lock, then sync.""" - from .core import ( + from ..core import ( ensure_project, do_outdated, do_lock, @@ -738,13 +476,13 @@ def update( project, ) - ensure_project(three=three, python=python, warn=True, pypi_mirror=pypi_mirror) + ensure_project(three=state.three, python=state.python, warn=True, pypi_mirror=state.pypi_mirror) if not outdated: outdated = bool(dry_run) if outdated: - do_outdated(pypi_mirror=pypi_mirror) - packages = [p for p in packages if p] - editable = [p for p in editable if p] + do_outdated(pypi_mirror=state.pypi_mirror) + packages = [p for p in state.packages if p] + editable = [p for p in state.editable if p] if not packages: echo( "{0} {1} {2} {3}{4}".format( @@ -766,51 +504,52 @@ def update( ), err=True, ) - ctx.abort + ctx.abort() do_lock( - clear=clear, - pre=pre, - keep_outdated=keep_outdated, - pypi_mirror=pypi_mirror, + clear=state.clear, + pre=state.installstate.pre, + keep_outdated=state.installstate.keep_outdated, + pypi_mirror=state.installstate.pypi_mirror, ) do_sync( ctx=ctx, - dev=dev, - three=three, - python=python, + dev=state.installstate.dev, + three=state.three, + python=state.python, bare=bare, - dont_upgrade=False, + dont_upgrade=not state.installstate.keep_outdated, user=False, - clear=clear, + clear=state.clear, unused=False, - sequential=sequential, - pypi_mirror=pypi_mirror, + sequential=state.installstate.sequential, + pypi_mirror=state.pypi_mirror, ) -@command(short_help=u"Displays currently-installed dependency graph information.") +@cli.command(short_help=u"Displays currently-installed dependency graph information.") @option("--bare", is_flag=True, default=False, help="Minimal output.") @option("--json", is_flag=True, default=False, help="Output JSON.") @option("--json-tree", is_flag=True, default=False, help="Output JSON in nested tree.") @option("--reverse", is_flag=True, default=False, help="Reversed dependency graph.") def graph(bare=False, json=False, json_tree=False, reverse=False): """Displays currently-installed dependency graph information.""" - from .core import do_graph + from ..core import do_graph do_graph(bare=bare, json=json, json_tree=json_tree, reverse=reverse) -@command(short_help="View a given module in your editor.", name="open") -@python_group -@pypi_mirror +@cli.command(short_help="View a given module in your editor.", name="open") +@common_options @argument("module", nargs=1) -def run_open(module, three=None, python=None, pypi_mirror=None): +@pass_state +def run_open(state, module, *args, **kwargs): """View a given module in your editor.""" - from .core import which, ensure_project + from ..core import which, ensure_project # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) + ensure_project(three=state.three, python=state.python, validate=False, + pypi_mirror=state.pypi_mirror) c = delegator.run( '{0} -c "import {1}; print({1}.__file__);"'.format(which("python"), module) ) @@ -825,88 +564,55 @@ def run_open(module, three=None, python=None, pypi_mirror=None): p = c.out.strip().rstrip("cdo") echo(crayons.normal("Opening {0!r} in your EDITOR.".format(p), bold=True)) edit(filename=p) - sys.exit(0) + return 0 -@command(short_help="Installs all packages specified in Pipfile.lock.") -@common_group -@install_group -@python_group +@cli.command(short_help="Installs all packages specified in Pipfile.lock.") @option("--bare", is_flag=True, default=False, help="Minimal output.") -@option("--clear", is_flag=True, default=False, help="Clear the dependency cache.") +@sync_options +@pass_state @pass_context def sync( ctx, - dev=False, - three=None, - python=None, + state, bare=False, - dont_upgrade=False, user=False, - clear=False, unused=False, - package_name=None, - sequential=False, - pypi_mirror=None, + **kwargs ): """Installs all packages specified in Pipfile.lock.""" - from .core import do_sync + from ..core import do_sync - do_sync( + retcode = do_sync( ctx=ctx, - dev=dev, - three=three, - python=python, + dev=state.installstate.dev, + three=state.three, + python=state.python, bare=bare, - dont_upgrade=dont_upgrade, + dont_upgrade=(not state.installstate.keep_outdated), user=user, - clear=clear, + clear=state.clear, unused=unused, - sequential=sequential, - pypi_mirror=pypi_mirror, + sequential=state.installstate.sequential, + pypi_mirror=state.pypi_mirror, ) + if retcode: + ctx.abort() -@command(short_help="Uninstalls all packages not specified in Pipfile.lock.") -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - help="Verbose mode.", - callback=setup_verbosity, -) -@python_group -@option( - "--dry-run", - is_flag=True, - default=False, - help="Just output unneeded packages.", -) +@cli.command(short_help="Uninstalls all packages not specified in Pipfile.lock.") +@option("--dry-run", is_flag=True, default=False, help="Just output unneeded packages.") +@verbose_option +@three_option +@python_option +@pass_state @pass_context -def clean(ctx, three=None, python=None, dry_run=False, bare=False, user=False): +def clean(ctx, state, dry_run=False, bare=False, user=False): """Uninstalls all packages not specified in Pipfile.lock.""" - from .core import do_clean - - do_clean( - ctx=ctx, - three=three, python=python, - dry_run=dry_run, - ) + from ..core import do_clean + do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run) -# Install click commands. -cli.add_command(graph) -cli.add_command(install) -cli.add_command(uninstall) -cli.add_command(sync) -cli.add_command(lock) -cli.add_command(check) -cli.add_command(clean) -cli.add_command(shell) -cli.add_command(run) -cli.add_command(update) -cli.add_command(run_open) # Only invoke the "did you mean" when an argument wasn't passed (it breaks those). if "-" not in "".join(sys.argv) and len(sys.argv) > 1: cli = DYMCommandCollection(sources=[cli]) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py new file mode 100644 index 0000000000..6441de523a --- /dev/null +++ b/pipenv/cli/options.py @@ -0,0 +1,368 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, unicode_literals +from click import Group, option, Option, make_pass_decorator, argument, BadParameter, BOOL as click_booltype, echo +from ..utils import is_valid_url +import os +from .. import environments + +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) + + +class PipenvGroup(Group): + """Custom Group class provides formatted main help""" + + def get_help_option(self, ctx): + from ..core import format_help + + """Override for showing formatted main help via --help and -h options""" + help_options = self.get_help_option_names(ctx) + if not help_options or not self.add_help_option: + return + + def show_help(ctx, param, value): + if value and not ctx.resilient_parsing: + if not ctx.invoked_subcommand: + # legit main help + echo(format_help(ctx.get_help())) + else: + # legit sub-command help + echo(ctx.get_help(), color=ctx.color) + ctx.exit() + + return Option( + help_options, + is_flag=True, + is_eager=True, + expose_value=False, + callback=show_help, + help="Show this message and exit.", + ) + + +class State(object): + def __init__(self): + self.index = None + self.extra_index_urls = [] + self.verbose = False + self.pypi_mirror = None + self.python = None + self.two = None + self.three = None + self.site_packages = False + self.clear = False + self.system = False + self.installstate = InstallState() + + +class InstallState(object): + def __init__(self): + self.dev = False + self.pre = False + self.selective_upgrade = False + self.keep_outdated = False + self.skip_lock = False + self.ignore_pipfile = False + self.sequential = False + self.code = False + self.requirementstxt = None + self.deploy = False + self.packages = [] + self.editables = [] + + +pass_state = make_pass_decorator(State, ensure=True) + + +def index_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.index = value + return value + return option('-i', '--index', expose_value=False, multiple=True, envvar="PIP_INDEX_URL", + help='Target PyPI-compatible package index url.', nargs=1, + callback=callback)(f) + + +def extra_index_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if isinstance(value, (tuple, list)): + state.extra_index_urls.extend(list(value)) + else: + state.extra_index_urls.append(value) + return value + return option("--extra-index-url", multiple=True, expose_value=False, + help=u"URLs to the extra PyPI compatible indexes to query for package lookups.", + callback=callback, envvar="PIP_EXTRA_INDEX_URL")(f) + + +def editable_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.editables.extend(value) + return value + return option('-e', '--editable', expose_value=False, multiple=True, + help='An editable python package URL or path, often to a VCS repo.', + callback=callback)(f) + + +def sequential_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.sequential = value + return value + return option("--sequential", is_flag=True, default=False, expose_value=False, + help="Install dependencies one-at-a-time, instead of concurrently.", + callback=callback, type=click_booltype)(f) + + +def skip_lock_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.skip_lock = value + return value + return option("--skip-lock", is_flag=True, default=False, expose_value=False, + help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", + callback=callback, type=click_booltype)(f) + + +def keep_outdated_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.keep_outdated = value + return value + return option("--keep-outdated", is_flag=True, default=False, expose_value=False, + help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", + callback=callback, type=click_booltype)(f) + + +def selective_upgrade_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.selective_upgrade = value + return value + return option("--selective-upgrade", is_flag=True, default=False, type=click_booltype, + help="Update specified packages.", callback=callback, + expose_value=False)(f) + + +def ignore_pipfile_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.ignore_pipfile = value + return value + return option("--ignore-pipfile", is_flag=True, default=False, expose_value=False, + help="Ignore Pipfile when installing, using the Pipfile.lock.", + callback=callback)(f) + + +def dev_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.dev = value + return value + return option("--dev", "-d", is_flag=True, default=False, type=click_booltype, + help="Install package(s) in [dev-packages].", callback=callback, + expose_value=False)(f) + + +def pre_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.pre = value + return value + return option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.", + callback=callback, type=click_booltype, expose_value=False)(f) + + +def package_arg(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.packages.extend(value) + return value + return argument('packages', nargs=-1, callback=callback, expose_value=False,)(f) + + +def three_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value is not None: + state.three = value + state.two = not value + return value + return option("--three/--two", is_flag=True, default=None, + help="Use Python 3/2 when creating virtualenv.", callback=callback, + expose_value=False)(f) + + +def python_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value is not None: + state.python = validate_python_path(ctx, param, value) + return value + return option("--python", default=False, nargs=1, callback=callback, + help="Specify which version of Python virtualenv should use.", + expose_value=False)(f) + + +def pypi_mirror_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value is not None: + state.pypi_mirror = validate_pypi_mirror(ctx, param, value) + return value + return option("--pypi-mirror", default=environments.PIPENV_PYPI_MIRROR, nargs=1, + callback=callback, help="Specify a PyPI mirror.", expose_value=False)(f) + + +def verbose_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + state.verbose = True + setup_verbosity(ctx, param, value) + return option("--verbose", "-v", is_flag=True, expose_value=False, + callback=callback, help="Verbose mode.")(f) + + +def site_packages_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.site_packages = value + return value + return option("--site-packages", is_flag=True, default=False, type=click_booltype, + help="Enable site-packages for the virtualenv.", callback=callback, + expose_value=False)(f) + + +def clear_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.clear = value + return value + return option("--clear", is_flag=True, callback=callback, type=click_booltype, + help="Clears caches (pipenv, pip, and pip-tools).", + expose_value=False)(f) + + +def system_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value is not None: + state.system = value + return value + return option("--system", is_flag=True, default=False, help="System pip management.", + callback=callback, type=click_booltype, expose_value=False)(f) + + +def requirementstxt_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + state.installstate.requirementstxt = value + return value + return option("--requirements", "-r", nargs=1, default=False, expose_value=False, + help="Import a requirements.txt file.", callback=callback)(f) + + +def code_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + state.installstate.code = value + return value + return option("--code", "-c", nargs=1, default=False, help="Import from codebase.", + callback=callback, expose_value=False)(f) + + +def deploy_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.deploy = value + return value + return option("--deploy", is_flag=True, default=False, type=click_booltype, + help=u"Abort if the Pipfile.lock is out-of-date, or Python version is" + " wrong.", callback=callback, expose_value=False)(f) + + +def setup_verbosity(ctx, param, value): + if not value: + return + import logging + logging.getLogger("pip").setLevel(logging.INFO) + environments.PIPENV_VERBOSITY = 1 + + +def validate_python_path(ctx, param, value): + # Validating the Python path is complicated by accepting a number of + # friendly options: the default will be boolean False to enable + # autodetection but it may also be a value which will be searched in + # the path or an absolute path. To report errors as early as possible + # we'll report absolute paths which do not exist: + if isinstance(value, (str, bytes)): + if os.path.isabs(value) and not os.path.isfile(value): + raise BadParameter("Expected Python at path %s does not exist" % value) + return value + + +def validate_pypi_mirror(ctx, param, value): + if value and not is_valid_url(value): + raise BadParameter("Invalid PyPI mirror URL: %s" % value) + return value + + +def common_options(f): + f = pypi_mirror_option(f) + f = verbose_option(f) + f = clear_option(f) + f = three_option(f) + f = python_option(f) + return f + + +def install_base_options(f): + f = common_options(f) + f = dev_option(f) + f = keep_outdated_option(f) + return f + + +def uninstall_options(f): + f = install_base_options(f) + f = skip_lock_option(f) + f = editable_option(f) + f = package_arg(f) + return f + + +def lock_options(f): + f = install_base_options(f) + f = index_option(f) + f = extra_index_option(f) + f = skip_lock_option(f) + f = pre_option(f) + return f + + +def sync_options(f): + f = install_base_options(f) + f = sequential_option(f) + f = sequential_option(f) + return f + + +def install_options(f): + f = lock_options(f) + f = sequential_option(f) + f = selective_upgrade_option(f) + f = ignore_pipfile_option(f) + f = editable_option(f) + f = package_arg(f) + return f + + +def general_options(f): + f = common_options(f) + f = site_packages_option(f) + return f diff --git a/pipenv/core.py b/pipenv/core.py index a338063f0c..3ccd8bc019 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -672,7 +672,7 @@ def cleanup_procs(procs, concurrent): click.echo( "{0} {1}! Will try again.".format( crayons.red("An error occurred while installing"), - crayons.green(c.dep.split("--hash")[0].strip()), + crayons.green(c.dep.as_line()), ) ) @@ -766,9 +766,6 @@ def cleanup_procs(procs, concurrent): ) for dep, ignore_hash in progress.bar(failed_deps_list, label=INSTALL_LABEL2): # Use a specific index, if specified. - dep, index = split_argument(dep, short="i", long_="index", num=1) - dep, extra_indexes = split_argument(dep, long_="extra-index-url") - dep = Requirement.from_line(dep) # Install the module. c = pip_install( dep, @@ -1287,7 +1284,7 @@ def pip_install( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir ) with os.fdopen(fd, "w") as f: - f.write(requirement.normalized_name) + f.write(requirement.as_line()) # Install dependencies when a package is a VCS dependency. if requirement and requirement.vcs: no_deps = False @@ -1935,6 +1932,8 @@ def do_uninstall( if PIPENV_USE_SYSTEM: system = True # Ensure that virtualenv is available. + # TODO: We probably shouldn't ensure a project exists if the outcome will be to just + # install things in order to remove them... maybe tell the user to install first? ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) editable_pkgs = [ Requirement.from_line("-e {0}".format(p)).name diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index c04dc3f679..9fad7d471a 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -344,7 +344,7 @@ def test_editable_no_args(PipenvInstance): with PipenvInstance() as p: c = p.pipenv("install -e") assert c.return_code != 0 - assert "Please provide path to editable package" in c.err + assert "Error: -e option requires an argument" in c.err @pytest.mark.install diff --git a/tests/integration/test_sync.py b/tests/integration/test_sync.py index 401eadfe04..c50e259b49 100644 --- a/tests/integration/test_sync.py +++ b/tests/integration/test_sync.py @@ -7,7 +7,7 @@ @pytest.mark.sync def test_sync_error_without_lockfile(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance(pypi=pypi, chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] @@ -40,7 +40,7 @@ def test_mirror_lock_sync(PipenvInstance, pypi): def test_sync_should_not_lock(PipenvInstance, pypi): """Sync should not touch the lock file, even if Pipfile is changed. """ - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance(pypi=pypi, chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] From c3b6143526e88e0c432dec82370baf9b7e561ab8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 2 Sep 2018 01:20:51 -0400 Subject: [PATCH 06/21] Cleanup unicode literals warnings Signed-off-by: Dan Ryan --- pipenv/cli/__init__.py | 2 +- pipenv/cli/command.py | 2 +- pipenv/cli/options.py | 13 ++++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pipenv/cli/__init__.py b/pipenv/cli/__init__.py index 1064ee3b14..605f4c10ec 100644 --- a/pipenv/cli/__init__.py +++ b/pipenv/cli/__init__.py @@ -1,3 +1,3 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import from .command import cli diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index d7815d5f52..e9848e95ca 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import import os import sys diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 6441de523a..8320bc0e85 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -1,9 +1,16 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals -from click import Group, option, Option, make_pass_decorator, argument, BadParameter, BOOL as click_booltype, echo -from ..utils import is_valid_url +from __future__ import absolute_import + import os + +from click import BOOL as click_booltype +from click import ( + BadParameter, Group, Option, argument, echo, make_pass_decorator, option +) + from .. import environments +from ..utils import is_valid_url + CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) From 305628e458a648b1781d66b12a6d199c5c400288 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 2 Sep 2018 13:06:07 -0400 Subject: [PATCH 07/21] Fix some typos Signed-off-by: Dan Ryan --- pipenv/cli/command.py | 17 +++++++---------- pipenv/cli/options.py | 2 +- pipenv/core.py | 9 +++++---- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index e9848e95ca..b82c9519ae 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -193,7 +193,7 @@ def cli( ) ctx.abort() # --two / --three was passed… - if (python or three is not None) or site_packages: + if (state.python or state.three is not None) or site_packages: ensure_project( three=state.three, python=state.python, @@ -319,7 +319,7 @@ def lock( pypi_mirror=state.pypi_mirror) do_lock( clear=state.clear, - pre=state.pre, + pre=state.installstate.pre, keep_outdated=state.installstate.keep_outdated, pypi_mirror=state.pypi_mirror, ) @@ -348,12 +348,9 @@ def lock( @pass_state def shell( state, - three=None, - python=False, fancy=False, shell_args=None, anyway=False, - pypi_mirror=None, ): """Spawns a shell within the virtualenv.""" from ..core import load_dot_env, do_shell @@ -378,11 +375,11 @@ def shell( if os.name == "nt": fancy = True do_shell( - three=three, - python=python, + three=state.three, + python=state.python, fancy=fancy, shell_args=shell_args, - pypi_mirror=pypi_mirror, + pypi_mirror=state.pypi_mirror, ) @@ -481,8 +478,8 @@ def update( outdated = bool(dry_run) if outdated: do_outdated(pypi_mirror=state.pypi_mirror) - packages = [p for p in state.packages if p] - editable = [p for p in state.editable if p] + packages = [p for p in state.installstate.packages if p] + editable = [p for p in state.installstate.editables if p] if not packages: echo( "{0} {1} {2} {3}{4}".format( diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 8320bc0e85..3b1d3d0246 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -85,7 +85,7 @@ def callback(ctx, param, value): state = ctx.ensure_object(State) state.index = value return value - return option('-i', '--index', expose_value=False, multiple=True, envvar="PIP_INDEX_URL", + return option('-i', '--index', expose_value=False, envvar="PIP_INDEX_URL", help='Target PyPI-compatible package index url.', nargs=1, callback=callback)(f) diff --git a/pipenv/core.py b/pipenv/core.py index 3ccd8bc019..421aac55dc 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1634,7 +1634,7 @@ def do_install( ) if selective_upgrade: keep_outdated = True - packages = packages if packages else [] + packages = packages if packages else[] editable_packages = editable_packages if editable_packages else [] package_args = [p for p in packages if p] + [p for p in editable_packages if p] skip_requirements = False @@ -1814,11 +1814,12 @@ def do_install( 'packages': [(pkg, pkg) for pkg in packages], 'editables': [("-e {0}".format(pkg), pkg) for pkg in editable_packages] } + pkg_tuples = [pkg_tuple for pkg_list in pkg_dict.values() for pkg_tuple in pkg_list] - for pkg_type, pkg_tuple in pkg_dict.items(): + for pkg_tuple in pkg_tuples: if not pkg_tuple: continue - pkg_line, pkg_val = pkg_tuple.pop() + pkg_line, pkg_val = pkg_tuple click.echo( crayons.normal( u"Installing {0}…".format(crayons.green(pkg_line, bold=True)), @@ -1965,7 +1966,7 @@ def do_uninstall( ) ) package_names = project.dev_packages.keys() - if packages is False and editable_packages is False and not all_dev: + if not packages and not editable_packages and not all_dev: click.echo(crayons.red("No package provided!"), err=True) return 1 for package_name in package_names: From 12217e69dcc61747b32b0a81aa3d8ae66651040f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 2 Sep 2018 18:37:26 -0400 Subject: [PATCH 08/21] Clean up vcs ref checkouts Signed-off-by: Dan Ryan --- pipenv/utils.py | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 12750fdac4..1c026e3ad8 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1167,22 +1167,6 @@ def extract_uri_from_vcs_dep(dep): return None -def resolve_ref(vcs_obj, target_dir, ref): - return vcs_obj.get_revision_sha(target_dir, ref) - - -def obtain_vcs_req(vcs_obj, src_dir, name, rev=None): - target_dir = os.path.join(src_dir, name) - target_rev = vcs_obj.make_rev_options(rev) - if not os.path.exists(target_dir): - vcs_obj.obtain(target_dir) - if not vcs_obj.is_commit_id_equal( - target_dir, rev - ) and not vcs_obj.is_commit_id_equal(target_dir, target_rev): - vcs_obj.update(target_dir, target_rev) - return vcs_obj.get_revision(target_dir) - - def get_vcs_deps( project, pip_freeze=None, @@ -1193,7 +1177,6 @@ def get_vcs_deps( dev=False, pypi_mirror=None, ): - from .patched.notpip._internal.vcs import VcsSupport from ._compat import TemporaryDirectory, Path from .vendor.requirementslib import Requirement @@ -1212,18 +1195,11 @@ def get_vcs_deps( os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) ) src_dir.mkdir(mode=0o775, exist_ok=True) - vcs_registry = VcsSupport for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) - backend = vcs_registry()._registry.get(requirement.vcs) - __vcs = backend(url=requirement.req.vcs_uri) - locked_rev = None name = requirement.normalized_name - locked_rev = obtain_vcs_req( - __vcs, src_dir.as_posix(), name, rev=pkg_pipfile.get("ref") - ) if requirement.is_vcs: - requirement.req.ref = locked_rev + requirement.req.lock_vcs_ref() lockfile[name] = requirement.pipfile_entry[1] reqs.append(requirement) return reqs, lockfile From 597bc8c88f8a67b1aadaedb01fd64d95267b0e06 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 2 Sep 2018 18:39:41 -0400 Subject: [PATCH 09/21] Update requirementslib Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/__init__.py | 2 +- pipenv/vendor/requirementslib/models/cache.py | 163 +++++++++++++----- .../vendor/requirementslib/models/markers.py | 6 +- .../vendor/requirementslib/models/pipfile.py | 10 +- .../requirementslib/models/requirements.py | 31 +++- pipenv/vendor/requirementslib/models/utils.py | 61 +++++-- 6 files changed, 197 insertions(+), 76 deletions(-) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 910db3d516..8eb90d547e 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.1.5' +__version__ = '1.1.6.dev0' from .exceptions import RequirementError diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 9b6d32e95c..16fc4ba8af 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -2,25 +2,20 @@ from __future__ import absolute_import, print_function, unicode_literals import copy -import errno import hashlib import json import os import six import sys -from contextlib import contextmanager - import requests +import pip_shims import vistir from appdirs import user_cache_dir from packaging.requirements import Requirement -from pip_shims.shims import ( - FAVORITE_HASH, Link, SafeFileCache, VcsSupport, is_file_url, url_to_path -) -from .utils import as_tuple, key_from_req, lookup_table +from .utils import as_tuple, key_from_req, lookup_table, get_pinned_version if six.PY2: @@ -194,22 +189,24 @@ def _reverse_dependencies(self, cache_keys): for dep_name in self.cache[name][version_and_extras]) -class HashCache(SafeFileCache): - """Caches hashes of PyPI artifacts so we do not need to re-download them +class HashCache(pip_shims.SafeFileCache): + """Caches hashes of PyPI artifacts so we do not need to re-download them. - Hashes are only cached when the URL appears to contain a hash in it and the cache key includes - the hash value returned from the server). This ought to avoid ssues where the location on the - server changes.""" + Hashes are only cached when the URL appears to contain a hash in it and the + cache key includes the hash value returned from the server). This ought to + avoid ssues where the location on the server changes. + """ def __init__(self, *args, **kwargs): session = kwargs.pop('session', requests.session()) + cache_dir = kwargs.pop('cache_dir', CACHE_DIR) self.session = session - kwargs.setdefault('directory', os.path.join(CACHE_DIR, 'hash-cache')) + kwargs.setdefault('directory', os.path.join(cache_dir, 'hash-cache')) super(HashCache, self).__init__(*args, **kwargs) def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs = VcsSupport() + vcs = pip_shims.VcsSupport() orig_scheme = location.scheme new_location = copy.deepcopy(location) if orig_scheme in vcs.all_schemes: @@ -217,7 +214,7 @@ def get_hash(self, location): can_hash = new_location.hash if can_hash: # hash url WITH fragment - hash_value = self.get(new_location.url) + hash_value = self._get_file_hash(new_location.url) if not new_location.url.startswith("ssh") else None if not hash_value: hash_value = self._get_file_hash(new_location) hash_value = hash_value.encode('utf8') @@ -226,41 +223,117 @@ def get_hash(self, location): return hash_value.decode('utf8') def _get_file_hash(self, location): - h = hashlib.new(FAVORITE_HASH) - with open_local_or_remote_file(location, self.session) as fp: + h = hashlib.new(pip_shims.FAVORITE_HASH) + with vistir.contextmanagers.open_file(location, self.session) as fp: for chunk in iter(lambda: fp.read(8096), b""): h.update(chunk) - return ":".join([FAVORITE_HASH, h.hexdigest()]) + return ":".join([pip_shims.FAVORITE_HASH, h.hexdigest()]) -@contextmanager -def open_local_or_remote_file(link, session): - """ - Open local or remote file for reading. +class _JSONCache(object): + """A persistent cache backed by a JSON file. + + The cache file is written to the appropriate user cache dir for the + current platform, i.e. + + ~/.cache/pip-tools/depcache-pyX.Y.json - :type link: pip._internal.index.Link - :type session: requests.Session - :raises ValueError: If link points to a local directory. - :return: a context manager to the opened file-like object + Where X.Y indicates the Python version. """ - if isinstance(link, Link): - url = link.url_without_fragment - else: - url = link - - if is_file_url(link): - # Local URL - local_path = url_to_path(url) - if os.path.isdir(local_path): - raise ValueError("Cannot open directory for read: {}".format(url)) + filename_format = None + + def __init__(self, cache_dir=CACHE_DIR): + vistir.mkdir_p(cache_dir) + python_version = ".".join(str(digit) for digit in sys.version_info[:2]) + cache_filename = self.filename_format.format( + python_version=python_version, + ) + self._cache_file = os.path.join(cache_dir, cache_filename) + self._cache = None + + @property + def cache(self): + """The dictionary that is the actual in-memory cache. + + This property lazily loads the cache from disk. + """ + if self._cache is None: + self.read_cache() + return self._cache + + def as_cache_key(self, ireq): + """Given a requirement, return its cache key. + + This behavior is a little weird in order to allow backwards + compatibility with cache files. For a requirement without extras, this + will return, for example:: + + ("ipython", "2.1.0") + + For a requirement with extras, the extras will be comma-separated and + appended to the version, inside brackets, like so:: + + ("ipython", "2.1.0[nbconvert,notebook]") + """ + extras = tuple(sorted(ireq.extras)) + if not extras: + extras_string = "" else: - with open(local_path, 'rb') as local_file: - yield local_file - else: - # Remote URL - headers = {"Accept-Encoding": "identity"} - response = session.get(url, headers=headers, stream=True) + extras_string = "[{}]".format(",".join(extras)) + name = key_from_req(ireq.req) + version = get_pinned_version(ireq) + return name, "{}{}".format(version, extras_string) + + def read_cache(self): + """Reads the cached contents into memory. + """ + if os.path.exists(self._cache_file): + self._cache = read_cache_file(self._cache_file) + else: + self._cache = {} + + def write_cache(self): + """Writes the cache to disk as JSON. + """ + doc = { + '__format__': 1, + 'dependencies': self._cache, + } + with open(self._cache_file, 'w') as f: + json.dump(doc, f, sort_keys=True) + + def clear(self): + self._cache = {} + self.write_cache() + + def __contains__(self, ireq): + pkgname, pkgversion_and_extras = self.as_cache_key(ireq) + return pkgversion_and_extras in self.cache.get(pkgname, {}) + + def __getitem__(self, ireq): + pkgname, pkgversion_and_extras = self.as_cache_key(ireq) + return self.cache[pkgname][pkgversion_and_extras] + + def __setitem__(self, ireq, values): + pkgname, pkgversion_and_extras = self.as_cache_key(ireq) + self.cache.setdefault(pkgname, {}) + self.cache[pkgname][pkgversion_and_extras] = values + self.write_cache() + + def __delitem__(self, ireq): + pkgname, pkgversion_and_extras = self.as_cache_key(ireq) try: - yield response.raw - finally: - response.close() + del self.cache[pkgname][pkgversion_and_extras] + except KeyError: + return + self.write_cache() + + def get(self, ireq, default=None): + pkgname, pkgversion_and_extras = self.as_cache_key(ireq) + return self.cache.get(pkgname, {}).get(pkgversion_and_extras, default) + + +class RequiresPythonCache(_JSONCache): + """Cache a candidate's Requires-Python information. + """ + filename_format = "pyreqcache-py{python_version}.json" diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/vendor/requirementslib/models/markers.py index 534978ed40..83b44b6344 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/vendor/requirementslib/models/markers.py @@ -82,12 +82,12 @@ def from_pipfile(cls, name, pipfile): marker_strings = ["{0} {1}".format(k, pipfile[k]) for k in found_keys] if pipfile.get("markers"): marker_strings.append(pipfile.get("markers")) - markers = [] + markers = set() for marker in marker_strings: - markers.append(marker) + markers.add(marker) combined_marker = None try: - combined_marker = cls.make_marker(" and ".join(markers)) + combined_marker = cls.make_marker(" and ".join(sorted(markers))) except RequirementError: pass else: diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 2bfd8996bc..3a6f5b1ee8 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -import os - from vistir.compat import Path from .requirements import Requirement @@ -30,10 +28,10 @@ def load(cls, path): with pipfile_path.open(encoding="utf-8") as fp: pipfile = super(Pipfile, cls).load(fp) pipfile.dev_requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.dev_packages.items() + Requirement.from_pipfile(k, v) for k, v in pipfile.get("dev-packages", {}).items() ] pipfile.requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.packages.items() + Requirement.from_pipfile(k, v) for k, v in pipfile.get("packages", {}).items() ] pipfile.path = pipfile_path return pipfile @@ -57,10 +55,10 @@ def load(cls, path): def dev_packages(self, as_requirements=True): if as_requirements: return self.dev_requirements - return self.dev_packages + return self.get('dev-packages', {}) @property def packages(self, as_requirements=True): if as_requirements: return self.requirements - return self.packages + return self.get('packages', {}) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 03c351847a..3b66bcd47a 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -517,8 +517,11 @@ def vcs_uri(self): return uri def get_commit_hash(self, src_dir=None): + is_local = False + if is_file_url(self.uri): + is_local = True src_dir = os.environ.get('SRC_DIR', None) if not src_dir else src_dir - if not src_dir: + if not src_dir and not is_local: _src_dir = TemporaryDirectory() atexit.register(_src_dir.cleanup) src_dir = _src_dir.name @@ -530,12 +533,16 @@ def get_commit_hash(self, src_dir=None): checkout_directory=checkout_dir, vcs_type=self.vcs ) - vcsrepo.obtain() + if not is_local: + vcsrepo.obtain() return vcsrepo.get_commit_hash() def update_repo(self, src_dir=None, ref=None): + is_local = False + if is_file_url(self.uri): + is_local = True src_dir = os.environ.get('SRC_DIR', None) if not src_dir else src_dir - if not src_dir: + if not src_dir and not is_local: _src_dir = TemporaryDirectory() atexit.register(_src_dir.cleanup) src_dir = _src_dir.name @@ -548,12 +555,17 @@ def update_repo(self, src_dir=None, ref=None): checkout_directory=checkout_dir, vcs_type=self.vcs ) - if not os.path.exists(checkout_dir): - vcsrepo.obtain() - else: - vcsrepo.update() + if not is_local: + if not not os.path.exists(checkout_dir): + vcsrepo.obtain() + else: + vcsrepo.update() return vcsrepo.get_commit_hash() + def lock_vcs_ref(self): + self.ref = self.get_commit_hash() + self.req.revision = self.ref + @req.default def get_requirement(self): name = self.name or self.link.egg_fragment @@ -884,12 +896,13 @@ def from_pipfile(cls, name, pipfile): def as_line(self, sources=None, include_hashes=True, include_extras=True): """Format this requirement as a line in requirements.txt. - If `sources` provided, it should be an sequence of mappings, containing + If ``sources`` provided, it should be an sequence of mappings, containing all possible sources to be used for this requirement. - If `sources` is omitted or falsy, no index information will be included + If ``sources`` is omitted or falsy, no index information will be included in the requirement line. """ + include_specifiers = True if self.specifiers else False if self.is_vcs: include_extras = False diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 6fd55b6ff2..6320236ee9 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -117,7 +117,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pipenv.patched.notpip format""" + """Cleans VCS uris from pip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: @@ -154,7 +154,7 @@ def validate_vcs(instance, attr_, value): def validate_path(instance, attr_, value): if not os.path.exists(value): - raise ValueError("Invalid path {0!r}", format(value)) + raise ValueError("Invalid path {0!r}".format(value)) def validate_markers(instance, attr_, value): @@ -256,9 +256,8 @@ def format_specifier(ireq): return ','.join(str(s) for s in specs) or '' -def is_pinned_requirement(ireq): - """ - Returns whether an InstallRequirement is a "pinned" requirement. +def get_pinned_version(ireq): + """Get the pinned version of an InstallRequirement. An InstallRequirement is considered pinned if: @@ -272,18 +271,54 @@ def is_pinned_requirement(ireq): django>1.8 # NOT pinned django~=1.8 # NOT pinned django==1.* # NOT pinned + + Raises `TypeError` if the input is not a valid InstallRequirement, or + `ValueError` if the InstallRequirement is not pinned. """ - if ireq.editable: - return False + try: + specifier = ireq.specifier + except AttributeError: + raise TypeError("Expected InstallRequirement, not {}".format( + type(ireq).__name__, + )) - specifier = getattr(ireq, "specifier", None) + if ireq.editable: + raise ValueError("InstallRequirement is editable") if not specifier: - return False + raise ValueError("InstallRequirement has no version specification") if len(specifier._specs) != 1: - return False + raise ValueError("InstallRequirement has multiple specifications") + + op, version = next(iter(specifier._specs))._spec + if op not in ('==', '===') or version.endswith('.*'): + raise ValueError("InstallRequirement not pinned (is {0!r})".format( + op + version, + )) + + return version + - op, version = first(specifier._specs)._spec - return (op == '==' or op == '===') and not version.endswith('.*') +def is_pinned_requirement(ireq): + """Returns whether an InstallRequirement is a "pinned" requirement. + + An InstallRequirement is considered pinned if: + + - Is not editable + - It has exactly one specifier + - That specifier is "==" + - The version does not contain a wildcard + + Examples: + django==1.8 # pinned + django>1.8 # NOT pinned + django~=1.8 # NOT pinned + django==1.* # NOT pinned + """ + try: + get_pinned_version(ireq) + except (TypeError, ValueError): + return False + return True def as_tuple(ireq): @@ -473,3 +508,5 @@ def fix_requires_python_marker(requires_python): ]) marker_to_add = PackagingRequirement('fakepkg; {0}'.format(marker_str)).marker return marker_to_add + + From f93cefa354b336ae9ba834c913992cf300081e46 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 2 Sep 2018 19:56:05 -0400 Subject: [PATCH 10/21] Fix `lock -r` output to include all markers - Fixes #2748 Signed-off-by: Dan Ryan --- pipenv/cli/command.py | 13 +-- pipenv/cli/options.py | 22 +++-- pipenv/core.py | 23 ++--- pipenv/patched/piptools/repositories/pypi.py | 4 +- .../vendoring/patches/patched/piptools.patch | 96 +++++++++---------- 5 files changed, 76 insertions(+), 82 deletions(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index b82c9519ae..68532dda83 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -212,7 +212,6 @@ def cli( short_help="Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.", context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), ) -@requirementstxt_option @system_option @code_option @deploy_option @@ -295,18 +294,10 @@ def uninstall( @cli.command(short_help="Generates Pipfile.lock.") -@option( - "--requirements", - "-r", - is_flag=True, - default=False, - help="Generate output compatible with requirements.txt.", -) @lock_options @pass_state def lock( state, - requirements=False, **kwargs ): """Generates Pipfile.lock.""" @@ -314,8 +305,8 @@ def lock( # Ensure that virtualenv is available. ensure_project(three=state.three, python=state.python, pypi_mirror=state.pypi_mirror) - if requirements: - do_init(dev=state.installstate.dev, requirements=requirements, + if state.installstate.requirementstxt: + do_init(dev=state.installstate.dev, requirements=state.installstate.requirementstxt, pypi_mirror=state.pypi_mirror) do_lock( clear=state.clear, diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 3b1d3d0246..2d073288ce 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -273,6 +273,16 @@ def callback(ctx, param, value): help="Import a requirements.txt file.", callback=callback)(f) +def requirements_flag(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + state.installstate.requirementstxt = value + return value + return option("--requirements", "-r", default=False, is_flag=True, expose_value=False, + help="Generate output in requirements.txt format.", callback=callback)(f) + + def code_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) @@ -345,9 +355,7 @@ def uninstall_options(f): def lock_options(f): f = install_base_options(f) - f = index_option(f) - f = extra_index_option(f) - f = skip_lock_option(f) + f = requirements_flag(f) f = pre_option(f) return f @@ -355,13 +363,15 @@ def lock_options(f): def sync_options(f): f = install_base_options(f) f = sequential_option(f) - f = sequential_option(f) return f def install_options(f): - f = lock_options(f) - f = sequential_option(f) + f = sync_options(f) + f = index_option(f) + f = extra_index_option(f) + f = requirementstxt_option(f) + f = pre_option(f) f = selective_upgrade_option(f) f = ignore_pipfile_option(f) f = editable_option(f) diff --git a/pipenv/core.py b/pipenv/core.py index 421aac55dc..91323ca9e7 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -711,24 +711,18 @@ def cleanup_procs(procs, concurrent): ) failed_deps_list = [] if requirements: - # Comment out packages that shouldn't be included in - # requirements.txt, for pip9. - # Additional package selectors, specific to pip's --hash checking mode. - for l in (deps_list, dev_deps_list): - for i, dep in enumerate(l): - l[i] = list(l[i]) - if "--hash" in l[i][0]: - l[i][0] = l[i][0].split("--hash")[0].strip() index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") + deps_list = [dep for dep, ignore_hash, block in deps_list] + dev_deps_list = [dep for dep, ignore_hash, block in dev_deps_list] # Output only default dependencies click.echo(index_args) if not dev: - click.echo("\n".join(d[0] for d in sorted(deps_list))) + click.echo("\n".join(d.partition('--hash')[0].strip() for d in sorted(deps_list))) sys.exit(0) # Output only dev dependencies if dev: - click.echo("\n".join(d[0] for d in sorted(dev_deps_list))) + click.echo("\n".join(d.partition('--hash')[0].strip() for d in sorted(dev_deps_list))) sys.exit(0) procs = [] deps_list_bar = progress.bar( @@ -1634,7 +1628,7 @@ def do_install( ) if selective_upgrade: keep_outdated = True - packages = packages if packages else[] + packages = packages if packages else [] editable_packages = editable_packages if editable_packages else [] package_args = [p for p in packages if p] + [p for p in editable_packages if p] skip_requirements = False @@ -1814,12 +1808,11 @@ def do_install( 'packages': [(pkg, pkg) for pkg in packages], 'editables': [("-e {0}".format(pkg), pkg) for pkg in editable_packages] } - pkg_tuples = [pkg_tuple for pkg_list in pkg_dict.values() for pkg_tuple in pkg_list] - for pkg_tuple in pkg_tuples: + for pkg_type, pkg_tuple in pkg_dict.items(): if not pkg_tuple: continue - pkg_line, pkg_val = pkg_tuple + pkg_line, pkg_val = pkg_tuple.pop() click.echo( crayons.normal( u"Installing {0}…".format(crayons.green(pkg_line, bold=True)), @@ -1966,7 +1959,7 @@ def do_uninstall( ) ) package_names = project.dev_packages.keys() - if not packages and not editable_packages and not all_dev: + if packages is False and editable_packages is False and not all_dev: click.echo(crayons.red("No package provided!"), err=True) return 1 for package_name in package_names: diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 699db8e30d..9e74156055 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -294,9 +294,9 @@ def get_legacy_dependencies(self, ireq): except (TypeError, ValueError, AttributeError): pass else: - setup_requires = getattr(dist, "requires", None) + setup_requires = getattr(dist, "extras_require", None) if not setup_requires: - setup_requires = getattr(dist, "setup_requires", None) + setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} try: # Pip 9 and below reqset = RequirementSet( diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 6ae7011fb7..854d3de020 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -4,18 +4,18 @@ index 4e6174c..75f9b49 100644 +++ b/pipenv/patched/piptools/locations.py @@ -2,10 +2,13 @@ import os from shutil import rmtree - + from .click import secho -from ._compat import user_cache_dir +# Patch by vphilippon 2017-11-22: Use pipenv cache path. +# from ._compat import user_cache_dir +from pipenv.environments import PIPENV_CACHE_DIR - + # The user_cache_dir helper comes straight from pip itself -CACHE_DIR = user_cache_dir('pip-tools') +# CACHE_DIR = user_cache_dir(os.path.join('pip-tools')) +CACHE_DIR = PIPENV_CACHE_DIR - + # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py @@ -33,7 +33,7 @@ index 1c4b943..10447b6 100644 +import sys from contextlib import contextmanager from shutil import rmtree - + @@ -15,13 +16,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, @@ -43,7 +43,7 @@ index 1c4b943..10447b6 100644 + InstallRequirement, + SafeFileCache, ) - + -from ..cache import CACHE_DIR +from pip._vendor.packaging.requirements import Requirement +from pip._vendor.packaging.specifiers import SpecifierSet, Specifier @@ -58,12 +58,12 @@ index 1c4b943..10447b6 100644 + make_install_requirement, clean_requires_python) + from .base import BaseRepository - - + + @@ -37,6 +47,45 @@ except ImportError: from pip.wheel import WheelCache - - + + +class HashCache(SafeFileCache): + """Caches hashes of PyPI artifacts so we do not need to re-download them + @@ -105,7 +105,7 @@ index 1c4b943..10447b6 100644 + class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url - + @@ -46,10 +95,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. @@ -117,7 +117,7 @@ index 1c4b943..10447b6 100644 self.pip_options = pip_options - self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control) + self.wheel_cache = WheelCache(PIPENV_CACHE_DIR, pip_options.format_control) - + index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: @@ -74,11 +124,15 @@ class PyPIRepository(BaseRepository): @@ -128,20 +128,20 @@ index 1c4b943..10447b6 100644 + + # stores *full* path + fragment => sha256 + self._hash_cache = HashCache(session=session) - + # Setup file paths self.freshen_build_caches() - self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs')) - self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels')) + self._download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'pkgs')) + self._wheel_download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'wheels')) - + def freshen_build_caches(self): """ @@ -114,10 +168,14 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match - + - all_candidates = self.find_all_candidates(ireq.name) + all_candidates = clean_requires_python(self.find_all_candidates(ireq.name)) + @@ -152,12 +152,12 @@ index 1c4b943..10447b6 100644 prereleases=prereleases) + except TypeError: + matching_versions = [candidate.version for candidate in all_candidates] - + # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] @@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) - + # Turn the candidate into a pinned InstallRequirement - return make_install_requirement( - best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint @@ -211,7 +211,7 @@ index 1c4b943..10447b6 100644 + return set(self._json_dep_cache[ireq]) + except Exception: + return set() - + def get_dependencies(self, ireq): + json_results = set() + @@ -256,9 +256,9 @@ index 1c4b943..10447b6 100644 + except (TypeError, ValueError, AttributeError): + pass + else: -+ setup_requires = getattr(dist, "requires", None) ++ setup_requires = getattr(dist, "extras_require", None) + if not setup_requires: -+ setup_requires = getattr(dist, "setup_requires", None) ++ setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} try: - # Pip < 9 and below + # Pip 9 and below @@ -380,12 +380,12 @@ index 1c4b943..10447b6 100644 reqset.cleanup_files() + return set(self._dependencies_cache[ireq]) - + def get_hashes(self, ireq): @@ -210,6 +434,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() - + + vcs = VcsSupport() + if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + return set() @@ -412,13 +412,13 @@ index 1c4b943..10447b6 100644 + # matching_versions = list( + # ireq.specifier.filter((candidate.version for candidate in all_candidates))) + # matching_candidates = candidates_by_version[matching_versions[0]] - + return { - self._get_file_hash(candidate.location) + self._hash_cache.get_hash(candidate.location) for candidate in matching_candidates } - + - def _get_file_hash(self, location): - h = hashlib.new(FAVORITE_HASH) - with open_local_or_remote_file(location, self.session) as fp: @@ -435,11 +435,11 @@ index 05ec8fd..2f94f6b 100644 +++ b/pipenv/patched/piptools/resolver.py @@ -8,13 +8,14 @@ from itertools import chain, count import os - + from first import first +from pip._vendor.packaging.markers import default_environment from ._compat import InstallRequirement - + from . import click from .cache import DependencyCache from .exceptions import UnsupportedConstraint @@ -447,7 +447,7 @@ index 05ec8fd..2f94f6b 100644 -from .utils import (format_requirement, format_specifier, full_groupby, +from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) - + green = partial(click.style, fg='green') @@ -28,6 +29,7 @@ class RequirementSummary(object): def __init__(self, ireq): @@ -456,11 +456,11 @@ index 05ec8fd..2f94f6b 100644 + self.markers = ireq.markers self.extras = str(sorted(ireq.extras)) self.specifier = str(ireq.specifier) - + @@ -71,7 +73,7 @@ class Resolver(object): with self.repository.allow_all_wheels(): return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs} - + - def resolve(self, max_rounds=10): + def resolve(self, max_rounds=12): """ @@ -516,14 +516,14 @@ index 05ec8fd..2f94f6b 100644 + ireq.extras = ireq.extra elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) - + @@ -283,14 +301,14 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) - self.dependency_cache[ireq] = sorted(str(ireq.req) for ireq in dependencies) + self.dependency_cache[ireq] = sorted(format_requirement(_ireq) for _ireq in dependencies) - + # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] dependency_strings = self.dependency_cache[ireq] log.debug(' {:25} requires {}'.format(format_requirement(ireq), @@ -531,7 +531,7 @@ index 05ec8fd..2f94f6b 100644 for dependency_string in dependency_strings: - yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) + yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) - + def reverse_dependencies(self, ireqs): non_editable = [ireq for ireq in ireqs if not ireq.editable] diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py @@ -554,25 +554,25 @@ index fde5816..23a05f2 100644 @@ -2,6 +2,7 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) - + +import six import os import sys from itertools import chain, groupby @@ -11,13 +12,79 @@ from contextlib import contextmanager from ._compat import InstallRequirement - + from first import first - +from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier +from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from pip._vendor.packaging.markers import Marker, Op, Value, Variable from .click import style - - + + UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} - - + + +def simplify_markers(ireq): + """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" + @@ -642,8 +642,8 @@ index fde5816..23a05f2 100644 if ireq.req is None and ireq.link is not None: @@ -43,16 +110,51 @@ def comment(text): return style(text, fg='green') - - + + -def make_install_requirement(name, version, extras, constraint=False): +def make_install_requirement(name, version, extras, markers, constraint=False): # If no extras are specified, the extras string is blank @@ -651,7 +651,7 @@ index fde5816..23a05f2 100644 if extras: # Sort extras for stability extras_string = "[{}]".format(",".join(sorted(extras))) - + - return InstallRequirement.from_line( - str('{}{}=={}'.format(name, extras_string, version)), - constraint=constraint) @@ -693,8 +693,8 @@ index fde5816..23a05f2 100644 + parts.append("; {0}".format(requirement.marker)) + + return "".join(parts) - - + + def format_requirement(ireq, marker=None): @@ -63,10 +165,10 @@ def format_requirement(ireq, marker=None): if ireq.editable: @@ -702,14 +702,14 @@ index fde5816..23a05f2 100644 else: - line = str(ireq.req).lower() + line = _requirement_to_str_lowercase_name(ireq.req) - + - if marker: - line = '{} ; {}'.format(line, marker) + if marker and ';' not in line: + line = '{}; {}'.format(line, marker) - + return line - + diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index 7e8cdf3..0a0d27d 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py @@ -717,7 +717,7 @@ index 7e8cdf3..0a0d27d 100644 @@ -1,30 +1,42 @@ # -*- coding=utf-8 -*- import importlib - + -def do_import(module_path, subimport=None, old_path=None): + +def do_import(module_path, subimport=None, old_path=None, vendored_name=None): @@ -744,8 +744,8 @@ index 7e8cdf3..0a0d27d 100644 if subimport: return getattr(_tmp, subimport, _tmp) return _tmp -- - +- + -InstallRequirement = do_import('req.req_install', 'InstallRequirement') -parse_requirements = do_import('req.req_file', 'parse_requirements') -RequirementSet = do_import('req.req_set', 'RequirementSet') From f902d8f44276978d26eac87692021dad04a559e8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 2 Sep 2018 20:22:15 -0400 Subject: [PATCH 11/21] Turn off no-deps for tarballs/zips - Fixes #2173 Signed-off-by: Dan Ryan --- pipenv/core.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 91323ca9e7..ad1bf0b6fd 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -731,10 +731,21 @@ def cleanup_procs(procs, concurrent): for dep, ignore_hash, block in deps_list_bar: if len(procs) < PIPENV_MAX_SUBPROCESS: # Use a specific index, if specified. - dep, index = split_argument(dep, short="i", long_="index", num=1) - dep, extra_indexes = split_argument(dep, long_="extra-index-url") + index = None + if ' --index' in dep: + dep, _, index = dep.partition(' --index') + index = index.lstrip('=') + elif ' -i ' in dep: + dep, _, index = dep.partition(' -i ') + extra_indexes = [] + if '--extra-index-url' in dep: + split_dep = dep.split('--extra-index-url') + dep, extra_indexes = split_dep[0], split_dep[1:] dep = Requirement.from_line(dep) # Install the module. + prev_no_deps_setting = no_deps + if dep.is_file and any(dep.req.uri.endswith(ext) for ext in ['zip', 'tar.gz']): + no_deps = False c = pip_install( dep, ignore_hashes=ignore_hash, @@ -748,7 +759,10 @@ def cleanup_procs(procs, concurrent): ) c.dep = dep c.ignore_hash = ignore_hash + c.index = index + c.extra_indexes = extra_indexes procs.append(c) + no_deps = prev_no_deps_setting if len(procs) >= PIPENV_MAX_SUBPROCESS or len(procs) == len(deps_list): cleanup_procs(procs, concurrent) procs = [] @@ -761,15 +775,19 @@ def cleanup_procs(procs, concurrent): for dep, ignore_hash in progress.bar(failed_deps_list, label=INSTALL_LABEL2): # Use a specific index, if specified. # Install the module. + prev_no_deps_setting = no_deps + if dep.is_file and any(dep.req.uri.endswith(ext) for ext in ['zip', 'tar.gz']): + no_deps = False c = pip_install( dep, ignore_hashes=ignore_hash, allow_global=allow_global, no_deps=no_deps, - index=index, + index=c.index, requirements_dir=requirements_dir, - extra_indexes=extra_indexes, + extra_indexes=c.extra_indexes, ) + no_deps = prev_no_deps_setting # The Installation failed… if c.return_code != 0: # We echo both c.out and c.err because pip returns error details on out. From 9cfb863d1f3a06ebc61353a4a2a5c4a39d47629f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 2 Sep 2018 20:29:30 -0400 Subject: [PATCH 12/21] Syntax error... Signed-off-by: Dan Ryan Update requirementslib Signed-off-by: Dan Ryan --- pipenv/core.py | 4 +- .../requirementslib/models/requirements.py | 142 +++++++++++------- pipenv/vendor/requirementslib/models/vcs.py | 5 +- 3 files changed, 88 insertions(+), 63 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index ad1bf0b6fd..587c59a676 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -744,7 +744,7 @@ def cleanup_procs(procs, concurrent): dep = Requirement.from_line(dep) # Install the module. prev_no_deps_setting = no_deps - if dep.is_file and any(dep.req.uri.endswith(ext) for ext in ['zip', 'tar.gz']): + if dep.is_file_or_url and any(dep.req.uri.endswith(ext) for ext in ['zip', 'tar.gz']): no_deps = False c = pip_install( dep, @@ -776,7 +776,7 @@ def cleanup_procs(procs, concurrent): # Use a specific index, if specified. # Install the module. prev_no_deps_setting = no_deps - if dep.is_file and any(dep.req.uri.endswith(ext) for ext in ['zip', 'tar.gz']): + if dep.is_file_or_url and any(dep.req.uri.endswith(ext) for ext in ['zip', 'tar.gz']): no_deps = False c = pip_install( dep, diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 3b66bcd47a..0672d6db4e 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -475,6 +475,7 @@ class VCSRequirement(FileRequirement): # : vcs reference name (branch / commit / tag) ref = attr.ib(default=None) subdirectory = attr.ib(default=None) + _repo = attr.ib(default=None) name = attr.ib() link = attr.ib() req = attr.ib() @@ -516,56 +517,6 @@ def vcs_uri(self): uri = "{0}+{1}".format(self.vcs, uri) return uri - def get_commit_hash(self, src_dir=None): - is_local = False - if is_file_url(self.uri): - is_local = True - src_dir = os.environ.get('SRC_DIR', None) if not src_dir else src_dir - if not src_dir and not is_local: - _src_dir = TemporaryDirectory() - atexit.register(_src_dir.cleanup) - src_dir = _src_dir.name - checkout_dir = Path(src_dir).joinpath(self.name).as_posix() - vcsrepo = VCSRepository( - url=self.link.url, - name=self.name, - ref=self.ref if self.ref else None, - checkout_directory=checkout_dir, - vcs_type=self.vcs - ) - if not is_local: - vcsrepo.obtain() - return vcsrepo.get_commit_hash() - - def update_repo(self, src_dir=None, ref=None): - is_local = False - if is_file_url(self.uri): - is_local = True - src_dir = os.environ.get('SRC_DIR', None) if not src_dir else src_dir - if not src_dir and not is_local: - _src_dir = TemporaryDirectory() - atexit.register(_src_dir.cleanup) - src_dir = _src_dir.name - checkout_dir = Path(src_dir).joinpath(self.name).as_posix() - ref = self.ref if not ref else ref - vcsrepo = VCSRepository( - url=self.link.url, - name=self.name, - ref=ref if ref else None, - checkout_directory=checkout_dir, - vcs_type=self.vcs - ) - if not is_local: - if not not os.path.exists(checkout_dir): - vcsrepo.obtain() - else: - vcsrepo.update() - return vcsrepo.get_commit_hash() - - def lock_vcs_ref(self): - self.ref = self.get_commit_hash() - self.req.revision = self.ref - @req.default def get_requirement(self): name = self.name or self.link.egg_fragment @@ -598,6 +549,64 @@ def get_requirement(self): req.url = self.uri return req + @property + def is_local(self): + if is_file_url(self.uri): + return True + return False + + @property + def repo(self): + if self._repo is None: + self._repo = self.get_vcs_repo() + return self._repo + + def get_checkout_dir(self, src_dir=None): + src_dir = os.environ.get('PIP_SRC', None) if not src_dir else src_dir + checkout_dir = None + if self.is_local and self.editable: + path = self.path + if not path: + path = url_to_path(self.uri) + if path and os.path.exists(path): + checkout_dir = Path(self.path).absolute().as_posix() + return checkout_dir + return src_dir + + def get_vcs_repo(self, src_dir=None): + checkout_dir = self.get_checkout_dir(src_dir=src_dir) + if not checkout_dir: + _src_dir = TemporaryDirectory() + atexit.register(_src_dir.cleanup) + checkout_dir = Path(_src_dir.name).joinpath(self.name).absolute().as_posix() + url = "{0}#egg={1}".format(self.vcs_uri, self.name) + vcsrepo = VCSRepository( + url=url, + name=self.name, + ref=self.ref if self.ref else None, + checkout_directory=checkout_dir, + vcs_type=self.vcs + ) + if not (self.is_local and self.editable): + vcsrepo.obtain() + return vcsrepo + + def get_commit_hash(self): + hash_ = self.repo.get_commit_hash() + return hash_ + + def update_repo(self, src_dir=None, ref=None): + ref = self.ref if not ref else ref + if not (self.is_local and self.editable): + self.repo.update() + self.repo.checkout_ref(ref) + hash_ = self.repo.get_commit_hash() + return hash_ + + def lock_vcs_ref(self): + self.ref = self.get_commit_hash() + self.req.revision = self.ref + @classmethod def from_pipfile(cls, name, pipfile): creation_args = {} @@ -685,7 +694,7 @@ def _choose_vcs_source(pipfile): @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=filter_none).copy() + pipfile_dict = attr.asdict(self, filter=lambda k, v: v is not None and k.name != '_repo').copy() if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) name, _ = _strip_extras(pipfile_dict.pop("name")) @@ -714,12 +723,16 @@ def get_name(self): def requirement(self): return self.req.req - @property - def hashes_as_pip(self): + def get_hashes_as_pip(self, as_list=False): if self.hashes: + if as_list: + return [HASH_STRING.format(h) for h in self.hashes] return "".join([HASH_STRING.format(h) for h in self.hashes]) + return "" if not as_list else [] - return "" + @property + def hashes_as_pip(self): + self.get_hashes_as_pip() @property def markers_as_pip(self): @@ -893,7 +906,7 @@ def from_pipfile(cls, name, pipfile): cls_inst.req.req.line = cls_inst.as_line() return cls_inst - def as_line(self, sources=None, include_hashes=True, include_extras=True): + def as_line(self, sources=None, include_hashes=True, include_extras=True, as_list=False): """Format this requirement as a line in requirements.txt. If ``sources`` provided, it should be an sequence of mappings, containing @@ -914,15 +927,28 @@ def as_line(self, sources=None, include_hashes=True, include_extras=True): self.specifiers if include_specifiers else "", self.markers_as_pip, ] + if as_list: + # This is used for passing to a subprocess call + parts = ["".join(parts)] if include_hashes: - parts.append(self.hashes_as_pip) + hashes = self.get_hashes_as_pip(as_list=as_list) + if as_list: + parts.extend(hashes) + else: + parts.append(hashes) if sources and not (self.requirement.local_file or self.vcs): from ..utils import prepare_pip_source_args if self.index: sources = [s for s in sources if s.get("name") == self.index] - index_string = " ".join(prepare_pip_source_args(sources)) - parts.extend([" ", index_string]) + source_list = prepare_pip_source_args(sources) + if as_list: + parts.extend(sources) + else: + index_string = " ".join(source_list) + parts.extend([" ", index_string]) + if as_list: + return parts line = "".join(parts) return line diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index a588f62985..79b9d610eb 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -24,12 +24,11 @@ def get_repo_instance(self): def obtain(self): if not os.path.exists(self.checkout_directory): - self.repo_instance.obtain(self.checkout_directory) + self.repo_instance.unpack(self.checkout_directory) if self.ref: - self.checkout_ref(self.ref) + self.update(self.ref) self.commit_sha = self.get_commit_hash(self.ref) else: - self.ref = self.repo_instance.default_arg_rev if not self.commit_sha: self.commit_sha = self.get_commit_hash() From 661eb170fc30f3660eb5f0e23a36a0a9717faff6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Sep 2018 01:19:58 -0400 Subject: [PATCH 13/21] Reorganize pip installation to ditch custom parser Signed-off-by: Dan Ryan --- pipenv/core.py | 89 ++++++++++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 587c59a676..f2e81dd34e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -16,6 +16,7 @@ from blindspin import spinner import six +from .cmdparse import Script from .project import Project, SourceNotFound from .utils import ( convert_deps_to_pip, @@ -36,7 +37,6 @@ is_pinned, is_star, rmtree, - split_argument, fs_str, clean_resolved_dep, ) @@ -656,6 +656,7 @@ def do_install_dependencies( If requirements is True, simply spits out a requirements format to stdout. """ from .vendor.requirementslib import Requirement + def cleanup_procs(procs, concurrent): for c in procs: if concurrent: @@ -742,6 +743,15 @@ def cleanup_procs(procs, concurrent): split_dep = dep.split('--extra-index-url') dep, extra_indexes = split_dep[0], split_dep[1:] dep = Requirement.from_line(dep) + if index: + _index = None + try: + _index = project.find_source(index).get('name') + except SourceNotFound: + _index = None + dep.index = _index + dep._index = index + dep.extra_indexes = extra_indexes # Install the module. prev_no_deps_setting = no_deps if dep.is_file_or_url and any(dep.req.uri.endswith(ext) for ext in ['zip', 'tar.gz']): @@ -783,9 +793,9 @@ def cleanup_procs(procs, concurrent): ignore_hashes=ignore_hash, allow_global=allow_global, no_deps=no_deps, - index=c.index, + index=getattr(dep, "_index", None), requirements_dir=requirements_dir, - extra_indexes=c.extra_indexes, + extra_indexes=getattr(dep, "extra_indexes", None), ) no_deps = prev_no_deps_setting # The Installation failed… @@ -895,7 +905,7 @@ def parse_download_fname(fname, name): if fname.endswith(".tar"): fname, _ = os.path.splitext(fname) # Substring out package name (plus dash) from file name to get version. - version = fname[len(name) + 1 :] + version = fname[len(name) + 1:] # Ignore implicit post releases in version number. if "-" in version and version.split("-")[1].isdigit(): version = version.split("-")[0] @@ -1280,8 +1290,7 @@ def pip_install( pypi_mirror=None, ): from notpip._internal import logger as piplogger - from notpip._vendor.pyparsing import ParseException - from .vendor.requirementslib import Requirement + src = [] if environments.is_verbose(): piplogger.setLevel(logging.INFO) @@ -1302,13 +1311,7 @@ def pip_install( no_deps = False # Don't specify a source directory when using --system. if not allow_global and ("PIP_SRC" not in os.environ): - src = "--src {0}".format( - escape_grouped_arguments(project.virtualenv_src_location) - ) - else: - src = "" - else: - src = "" + src.extend(["--src", "{0}".format(project.virtualenv_src_location)]) # Try installing for each source in project.sources. if index: @@ -1336,37 +1339,39 @@ def pip_install( create_mirror_source(pypi_mirror) if is_pypi_url(source["url"]) else source for source in sources ] - if requirement and requirement.editable: - install_reqs = ' {0}'.format(requirement.as_line()) - elif r: - install_reqs = " -r {0}".format(escape_grouped_arguments(r)) + if (requirement and requirement.editable) or not r: + install_reqs = requirement.as_line(as_list=True) + if requirement.editable and install_reqs[0].startswith("-e "): + req, install_reqs = install_reqs[0], install_reqs[1:] + editable_opt, req = req.split(" ", 1) + install_reqs = [editable_opt, req] + install_reqs + if not any(item.startswith("--hash") for item in install_reqs): + ignore_hashes = True else: - install_reqs = ' "{0}"'.format(requirement.as_line()) - # Skip hash-checking mode, when appropriate. - if r: + install_reqs = ["-r", r] with open(r) as f: if "--hash" not in f.read(): ignore_hashes = True - else: - if "--hash" not in install_reqs: - ignore_hashes = True + pip_command = [ + which_pip(allow_global=allow_global), + "install" + ] + if pre: + pip_command.append("--pre") + if src: + pip_command.extend(src) + if environments.is_verbose(): + pip_command.append("--verbose") + pip_command.append("--upgrade") + if selective_upgrade: + pip_command.append("--upgrade-strategy=only-if-needed") + if no_deps: + pip_command.append("--no-deps") + pip_command.extend(install_reqs) + pip_command.extend(prepare_pip_source_args(sources)) if not ignore_hashes: - install_reqs += " --require-hashes" - pip_args = { - "no_deps": "--no-deps" if no_deps else "", - "pre": "--pre" if pre else "", - "quoted_pip": escape_grouped_arguments(which_pip(allow_global=allow_global)), - "upgrade_strategy": ( - "--upgrade --upgrade-strategy=only-if-needed" if selective_upgrade else "" - ), - "sources": " ".join(prepare_pip_source_args(sources)), - "src": src, - "verbose_flag": "--verbose" if environments.is_verbose() else "", - "install_reqs": install_reqs - } - pip_command = "{quoted_pip} install {pre} {src} {verbose_flag} {upgrade_strategy} {no_deps} {install_reqs} {sources}".format( - **pip_args - ) + pip_command.append("--require-hashes") + if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) cache_dir = Path(PIPENV_CACHE_DIR) @@ -1375,7 +1380,13 @@ def pip_install( "PIP_WHEEL_DIR": fs_str(cache_dir.joinpath("wheels").as_posix()), "PIP_DESTINATION_DIR": fs_str(cache_dir.joinpath("pkgs").as_posix()), "PIP_EXISTS_ACTION": fs_str("w"), + "PATH": fs_str(os.environ.get("PATH")) } + if src: + pip_config.update({ + "PIP_SRC": fs_str(project.virtualenv_src_location) + }) + pip_command = Script.parse(pip_command).cmdify() c = delegator.run(pip_command, block=block, env=pip_config) return c From eb0709b683902dc8365a9343bb1e88f75c63ee41 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Sep 2018 01:20:20 -0400 Subject: [PATCH 14/21] Fix installation of multiple packages Signed-off-by: Dan Ryan --- pipenv/core.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index f2e81dd34e..1f11ff7739 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1833,15 +1833,9 @@ def do_install( else: from .vendor.requirementslib import Requirement # make a tuple of (display_name, entry) - pkg_dict = { - 'packages': [(pkg, pkg) for pkg in packages], - 'editables': [("-e {0}".format(pkg), pkg) for pkg in editable_packages] - } + pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] - for pkg_type, pkg_tuple in pkg_dict.items(): - if not pkg_tuple: - continue - pkg_line, pkg_val = pkg_tuple.pop() + for pkg_line in pkg_list: click.echo( crayons.normal( u"Installing {0}…".format(crayons.green(pkg_line, bold=True)), @@ -1856,7 +1850,8 @@ def do_install( click.echo("{0}: {1}".format(crayons.red("WARNING"), e)) requirements_directory.cleanup() sys.exit(1) - + if index_url: + pkg_requirement.index = index_url c = pip_install( pkg_requirement, ignore_hashes=True, From ab2b3d3b5a98ed34478ca7c356f92883c6bde046 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Sep 2018 01:20:51 -0400 Subject: [PATCH 15/21] Minor project fixes Signed-off-by: Dan Ryan --- pipenv/project.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index c15dc4cf31..ff1f5364a5 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -289,7 +289,7 @@ def get_installed_packages(self): new_path = [new_path[0], PIPENV_ROOT, PIPENV_PATCHED, PIPENV_VENDOR] + new_path[1:] sys.path = new_path os.environ['VIRTUAL_ENV'] = self.virtualenv_location - from .patched.notpip._internal.utils.misc import get_installed_distributions + from .vendor.pip_shims.shims import get_installed_distributions return get_installed_distributions(local_only=True) else: return [] @@ -372,7 +372,10 @@ def virtualenv_location(self): @property def virtualenv_src_location(self): - loc = os.sep.join([self.virtualenv_location, "src"]) + if self.virtualenv_location: + loc = os.sep.join([self.virtualenv_location, "src"]) + else: + loc = os.sep.join([self.project_directory, "src"]) mkdir_p(loc) return loc From 86022ccd69476ad1d656682e5da4c88098b0a87c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Sep 2018 01:21:16 -0400 Subject: [PATCH 16/21] Add missing skip_lock option Signed-off-by: Dan Ryan --- pipenv/cli/command.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 68532dda83..f6c0d13385 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -19,7 +19,7 @@ from ..__version__ import __version__ from .options import ( CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, - general_options, install_options, lock_options, pass_state, + general_options, install_options, lock_options, pass_state, skip_lock_option, pypi_mirror_option, python_option, requirementstxt_option, sync_options, system_option, three_option, verbose_option, uninstall_options ) @@ -215,6 +215,7 @@ def cli( @system_option @code_option @deploy_option +@skip_lock_option @install_options @pass_state @pass_context @@ -498,7 +499,7 @@ def update( clear=state.clear, pre=state.installstate.pre, keep_outdated=state.installstate.keep_outdated, - pypi_mirror=state.installstate.pypi_mirror, + pypi_mirror=state.pypi_mirror, ) do_sync( ctx=ctx, From 2768edbc7d0920c4d4cb22877a5e01ec9ddba203 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Sep 2018 01:25:24 -0400 Subject: [PATCH 17/21] Fix editor auto-deletions of piptools patch lines Signed-off-by: Dan Ryan --- .../vendoring/patches/patched/piptools.patch | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 854d3de020..6b937337ed 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..10447b6 100644 +index 1c4b943..84077f0 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,9 +1,10 @@ @@ -33,7 +33,7 @@ index 1c4b943..10447b6 100644 +import sys from contextlib import contextmanager from shutil import rmtree - + @@ -15,13 +16,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, @@ -43,7 +43,7 @@ index 1c4b943..10447b6 100644 + InstallRequirement, + SafeFileCache, ) - + -from ..cache import CACHE_DIR +from pip._vendor.packaging.requirements import Requirement +from pip._vendor.packaging.specifiers import SpecifierSet, Specifier @@ -58,12 +58,12 @@ index 1c4b943..10447b6 100644 + make_install_requirement, clean_requires_python) + from .base import BaseRepository - - + + @@ -37,6 +47,45 @@ except ImportError: from pip.wheel import WheelCache - - + + +class HashCache(SafeFileCache): + """Caches hashes of PyPI artifacts so we do not need to re-download them + @@ -105,7 +105,7 @@ index 1c4b943..10447b6 100644 + class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url - + @@ -46,10 +95,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. @@ -117,7 +117,7 @@ index 1c4b943..10447b6 100644 self.pip_options = pip_options - self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control) + self.wheel_cache = WheelCache(PIPENV_CACHE_DIR, pip_options.format_control) - + index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: @@ -74,11 +124,15 @@ class PyPIRepository(BaseRepository): @@ -128,20 +128,20 @@ index 1c4b943..10447b6 100644 + + # stores *full* path + fragment => sha256 + self._hash_cache = HashCache(session=session) - + # Setup file paths self.freshen_build_caches() - self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs')) - self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels')) + self._download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'pkgs')) + self._wheel_download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'wheels')) - + def freshen_build_caches(self): """ @@ -114,10 +168,14 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match - + - all_candidates = self.find_all_candidates(ireq.name) + all_candidates = clean_requires_python(self.find_all_candidates(ireq.name)) + @@ -152,12 +152,12 @@ index 1c4b943..10447b6 100644 prereleases=prereleases) + except TypeError: + matching_versions = [candidate.version for candidate in all_candidates] - + # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] @@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) - + # Turn the candidate into a pinned InstallRequirement - return make_install_requirement( - best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint @@ -211,7 +211,7 @@ index 1c4b943..10447b6 100644 + return set(self._json_dep_cache[ireq]) + except Exception: + return set() - + def get_dependencies(self, ireq): + json_results = set() + @@ -380,12 +380,12 @@ index 1c4b943..10447b6 100644 reqset.cleanup_files() + return set(self._dependencies_cache[ireq]) - + def get_hashes(self, ireq): @@ -210,6 +434,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() - + + vcs = VcsSupport() + if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + return set() @@ -412,13 +412,13 @@ index 1c4b943..10447b6 100644 + # matching_versions = list( + # ireq.specifier.filter((candidate.version for candidate in all_candidates))) + # matching_candidates = candidates_by_version[matching_versions[0]] - + return { - self._get_file_hash(candidate.location) + self._hash_cache.get_hash(candidate.location) for candidate in matching_candidates } - + - def _get_file_hash(self, location): - h = hashlib.new(FAVORITE_HASH) - with open_local_or_remote_file(location, self.session) as fp: From e88d2d62c7a7f19875a733dbf262707f9e353c90 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Sep 2018 03:14:10 -0400 Subject: [PATCH 18/21] No idea why this doesn't work --- pipenv/utils.py | 62 ++++++--------------- pipenv/vendor/requirementslib/models/vcs.py | 21 +++++-- setup.cfg | 2 +- 3 files changed, 34 insertions(+), 51 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 1c026e3ad8..9d02f92be7 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1072,38 +1072,6 @@ def handle_remove_readonly(func, path, exc): raise -def split_argument(req, short=None, long_=None, num=-1): - """Split an argument from a string (finds None if not present). - - Uses -short , --long , and --long=arg as permutations. - - returns string, index - """ - index_entries = [] - import re - - if long_: - index_entries.append("--{0}".format(long_)) - if short: - index_entries.append("-{0}".format(short)) - match_string = "|".join(index_entries) - matches = re.findall("(?<=\s)({0})([\s=])(\S+)".format(match_string), req) - remove_strings = [] - match_values = [] - for match in matches: - match_values.append(match[-1]) - remove_strings.append("".join(match)) - for string_to_remove in remove_strings: - req = req.replace(" {0}".format(string_to_remove), "") - if not match_values: - return req, None - if num == 1: - return req, match_values[0] - if num == -1: - return req, match_values - return req, match_values[:num] - - @contextmanager def atomic_open_for_write(target, binary=False, newline=None, encoding=None): """Atomically open `target` for writing. @@ -1187,10 +1155,7 @@ def get_vcs_deps( packages = getattr(project, section) except AttributeError: return [], [] - if not os.environ.get("PIP_SRC") and not project.virtualenv_location: - _src_dir = TemporaryDirectory(prefix="pipenv-", suffix="-src") - src_dir = Path(_src_dir.name) - else: + if os.environ.get("PIP_SRC"): src_dir = Path( os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) ) @@ -1199,8 +1164,8 @@ def get_vcs_deps( requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) name = requirement.normalized_name if requirement.is_vcs: - requirement.req.lock_vcs_ref() lockfile[name] = requirement.pipfile_entry[1] + lockfile[name]['ref'] = requirement.req.repo.get_commit_hash() reqs.append(requirement) return reqs, lockfile @@ -1219,17 +1184,26 @@ def translate_markers(pipfile_entry): if not isinstance(pipfile_entry, Mapping): raise TypeError("Entry is not a pipfile formatted mapping.") from notpip._vendor.distlib.markers import DEFAULT_CONTEXT as marker_context + from .vendor.packaging.markers import Marker + from .vendor.vistir.misc import dedup allowed_marker_keys = ["markers"] + [k for k in marker_context.keys()] provided_keys = list(pipfile_entry.keys()) if hasattr(pipfile_entry, "keys") else [] - pipfile_marker = next((k for k in provided_keys if k in allowed_marker_keys), None) + pipfile_markers = [k for k in provided_keys if k in allowed_marker_keys] new_pipfile = dict(pipfile_entry).copy() - if pipfile_marker: - entry = "{0}".format(pipfile_entry[pipfile_marker]) - if pipfile_marker != "markers": - entry = "{0} {1}".format(pipfile_marker, entry) - new_pipfile.pop(pipfile_marker) - new_pipfile["markers"] = entry + marker_set = set() + if "markers" in new_pipfile: + marker_set.add(str(Marker(new_pipfile.get("markers")))) + for m in pipfile_markers: + entry = "{0}".format(pipfile_entry[m]) + if m != "markers": + marker_set.add(str(Marker("{0}'{1}'".format(m, entry)))) + new_pipfile.pop(m) + marker_set.add(str(entry)) + if marker_set: + new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) + if " and " in s else s + for s in sorted(dedup(marker_set))]))) return new_pipfile diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index 79b9d610eb..3bfb05d5fd 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- import attr -from pip_shims import VcsSupport +from pip_shims import VcsSupport, parse_version, pip_version +import vistir import os @@ -24,10 +25,11 @@ def get_repo_instance(self): def obtain(self): if not os.path.exists(self.checkout_directory): - self.repo_instance.unpack(self.checkout_directory) + self.repo_instance.obtain(self.checkout_directory) if self.ref: - self.update(self.ref) - self.commit_sha = self.get_commit_hash(self.ref) + with vistir.contextmanagers.cd(self.checkout_directory): + self.update(self.ref) + self.commit_sha = self.get_commit_hash() else: if not self.commit_sha: self.commit_sha = self.get_commit_hash() @@ -37,11 +39,18 @@ def checkout_ref(self, ref): if not self.repo_instance.is_commit_id_equal( self.checkout_directory, self.get_commit_hash(ref) ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref): - self.repo_instance.switch(self.checkout_directory, self.url, target_rev) + self.repo_instance.update(self.checkout_directory, self.url, target_rev) + self.commit_hash = self.get_commit_hash() def update(self, ref): target_rev = self.repo_instance.make_rev_options(ref) - self.repo_instance.update(self.checkout_directory, target_rev) + if parse_version(pip_version) > parse_version("18.0"): + self.repo_instance.update(self.checkout_directory, self.url, target_rev) + else: + self.repo_instance.update(self.checkout_directory, target_rev) + self.commit_hash = self.get_commit_hash() def get_commit_hash(self, ref=None): + if ref: + return self.repo_instance.get_revision(self.checkout_directory) return self.repo_instance.get_revision(self.checkout_directory) diff --git a/setup.cfg b/setup.cfg index f0cb3bf581..78140fa0c0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,6 @@ lines_between_types=1 multi_line_output=5 not_skip=__init__.py known_first_party = - passa + pipenv tests ignore_trailing_comma=true From b33dfa65667ed3291608575344b20cdf9b2ff47d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Sep 2018 19:03:57 -0400 Subject: [PATCH 19/21] Update requirementslib and fix VCS installation Signed-off-by: Dan Ryan Don't re-clone repos now that this works Signed-off-by: Dan Ryan Prune peeps directory from manifest Signed-off-by: Dan Ryan Fix nonetype uris Signed-off-by: Dan Ryan Manually lock requirements? Signed-off-by: Dan Ryan Update requirementslib - leave context before updating sha Signed-off-by: Dan Ryan Fix requirementslib vcs checkouts Signed-off-by: Dan Ryan fix tmpdir implementation Signed-off-by: Dan Ryan final fix for vcs uris Signed-off-by: Dan Ryan Allow for adding requirements objects directly to pipfile Signed-off-by: Dan Ryan Update piptools patch Signed-off-by: Dan Ryan --- MANIFEST.in | 2 + pipenv/_compat.py | 2 +- pipenv/core.py | 2 +- pipenv/project.py | 5 +- pipenv/utils.py | 19 ++-- .../requirementslib/models/requirements.py | 93 ++++++++++++------- pipenv/vendor/requirementslib/models/vcs.py | 29 ++++-- .../vendoring/patches/patched/piptools.patch | 6 +- tests/integration/test_install_uri.py | 8 +- 9 files changed, 107 insertions(+), 59 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index f593cf781c..e5cc05f2b1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,6 +24,8 @@ recursive-include docs/_templates *.html recursive-include docs/_static *.js *.css *.png recursive-exclude docs requirements*.txt + +prune peeps prune .buildkite prune .github prune .vsts-ci diff --git a/pipenv/_compat.py b/pipenv/_compat.py index b1861e4d21..558df3b846 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -90,7 +90,7 @@ class TemporaryDirectory(object): in it are removed. """ - def __init__(self, suffix=None, prefix=None, dir=None): + def __init__(self, suffix="", prefix="", dir=None): if "RAM_DISK" in os.environ: import uuid diff --git a/pipenv/core.py b/pipenv/core.py index 1f11ff7739..52c127b10a 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1908,7 +1908,7 @@ def do_install( ) # Add the package to the Pipfile. try: - project.add_package_to_pipfile(pkg_requirement.as_line(), dev) + project.add_package_to_pipfile(pkg_requirement, dev) except ValueError as e: click.echo( "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), e) diff --git a/pipenv/project.py b/pipenv/project.py index ff1f5364a5..f9decbf641 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -770,13 +770,14 @@ def remove_package_from_pipfile(self, package_name, dev=False): del p[key][name] self.write_toml(p) - def add_package_to_pipfile(self, package_name, dev=False): + def add_package_to_pipfile(self, package, dev=False): from .vendor.requirementslib import Requirement # Read and append Pipfile. p = self.parsed_pipfile # Don't re-capitalize file URLs or VCSs. - package = Requirement.from_line(package_name.strip()) + if not isinstance(package, Requirement): + package = Requirement.from_line(package.strip()) _, converted = package.pipfile_entry key = "dev-packages" if dev else "packages" # Set empty group if it doesn't exist yet. diff --git a/pipenv/utils.py b/pipenv/utils.py index 9d02f92be7..b7295c2921 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1146,6 +1146,7 @@ def get_vcs_deps( pypi_mirror=None, ): from ._compat import TemporaryDirectory, Path + import atexit from .vendor.requirementslib import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" @@ -1160,12 +1161,18 @@ def get_vcs_deps( os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) ) src_dir.mkdir(mode=0o775, exist_ok=True) + else: + src_dir = TemporaryDirectory(prefix="pipenv-lock-dir") + atexit.register(src_dir.cleanup) for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) name = requirement.normalized_name + commit_hash = None if requirement.is_vcs: - lockfile[name] = requirement.pipfile_entry[1] - lockfile[name]['ref'] = requirement.req.repo.get_commit_hash() + with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo: + commit_hash = repo.get_commit_hash() + lockfile[name] = requirement.pipfile_entry[1] + lockfile[name]['ref'] = commit_hash reqs.append(requirement) return reqs, lockfile @@ -1197,13 +1204,11 @@ def translate_markers(pipfile_entry): for m in pipfile_markers: entry = "{0}".format(pipfile_entry[m]) if m != "markers": - marker_set.add(str(Marker("{0}'{1}'".format(m, entry)))) + marker_set.add(str(Marker("{0}{1}".format(m, entry)))) new_pipfile.pop(m) - marker_set.add(str(entry)) if marker_set: - new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) - if " and " in s else s - for s in sorted(dedup(marker_set))]))) + new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) if " and " in s else s + for s in sorted(dedup(marker_set))]))).replace('"', "'") return new_pipfile diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 0672d6db4e..471ebcea14 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1,30 +1,35 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import atexit import collections import hashlib import os +from contextlib import contextmanager + import attr -import atexit from first import first from packaging.markers import Marker +from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from six.moves.urllib import parse as urllib_parse -from six.moves.urllib.parse import unquote - from pip_shims.shims import ( InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url, url_to_path ) +from six.moves.urllib import parse as urllib_parse +from six.moves.urllib.parse import unquote from vistir.compat import FileNotFoundError, Path, TemporaryDirectory from vistir.misc import dedup -from vistir.path import get_converted_relative_path, is_valid_url, is_file_url, mkdir_p +from vistir.path import ( + create_tracked_tempdir, get_converted_relative_path, is_file_url, + is_valid_url, mkdir_p +) from ..exceptions import RequirementError -from ..utils import VCS_LIST, is_vcs, is_installable_file +from ..utils import VCS_LIST, is_installable_file, is_vcs from .baserequirement import BaseRequirement from .dependencies import ( AbstractDependency, find_all_matches, get_abstract_dependencies, @@ -32,15 +37,14 @@ ) from .markers import PipenvMarkers from .utils import ( - HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, filter_none, - format_requirement, get_version, init_requirement, + HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string, + filter_none, format_requirement, get_version, init_requirement, is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, specs_to_string, split_markers_from_line, split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs, extras_to_string + validate_specifiers, validate_vcs ) from .vcs import VCSRepository -from packaging.requirements import Requirement as PackagingRequirement @attr.s @@ -481,6 +485,9 @@ class VCSRequirement(FileRequirement): req = attr.ib() def __attrs_post_init__(self): + if not self.uri: + if self.path: + self.uri = path_to_url(self.path) split = urllib_parse.urlsplit(self.uri) scheme, rest = split[0], split[1:] vcs_type = "" @@ -493,9 +500,10 @@ def __attrs_post_init__(self): @link.default def get_link(self): + uri = self.uri if self.uri else path_to_url(self.path) return build_vcs_link( self.vcs, - add_ssh_scheme_to_git_uri(self.uri), + add_ssh_scheme_to_git_uri(uri), name=self.name, ref=self.ref, subdirectory=self.subdirectory, @@ -564,21 +572,17 @@ def repo(self): def get_checkout_dir(self, src_dir=None): src_dir = os.environ.get('PIP_SRC', None) if not src_dir else src_dir checkout_dir = None - if self.is_local and self.editable: + if self.is_local: path = self.path if not path: path = url_to_path(self.uri) if path and os.path.exists(path): - checkout_dir = Path(self.path).absolute().as_posix() + checkout_dir = os.path.abspath(path) return checkout_dir - return src_dir + return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) def get_vcs_repo(self, src_dir=None): checkout_dir = self.get_checkout_dir(src_dir=src_dir) - if not checkout_dir: - _src_dir = TemporaryDirectory() - atexit.register(_src_dir.cleanup) - checkout_dir = Path(_src_dir.name).joinpath(self.name).absolute().as_posix() url = "{0}#egg={1}".format(self.vcs_uri, self.name) vcsrepo = VCSRepository( url=url, @@ -587,25 +591,36 @@ def get_vcs_repo(self, src_dir=None): checkout_directory=checkout_dir, vcs_type=self.vcs ) - if not (self.is_local and self.editable): + if not self.is_local: vcsrepo.obtain() return vcsrepo def get_commit_hash(self): + hash_ = None hash_ = self.repo.get_commit_hash() return hash_ def update_repo(self, src_dir=None, ref=None): - ref = self.ref if not ref else ref - if not (self.is_local and self.editable): - self.repo.update() + if ref: + self.ref = ref + else: + if self.ref: + ref = self.ref + repo_hash = None + if not self.is_local and ref is not None: self.repo.checkout_ref(ref) - hash_ = self.repo.get_commit_hash() - return hash_ - - def lock_vcs_ref(self): + repo_hash = self.repo.get_commit_hash() + return repo_hash + + @contextmanager + def locked_vcs_repo(self, src_dir=None): + vcsrepo = self.get_vcs_repo(src_dir=src_dir) + if self.ref and not self.is_local: + vcsrepo.checkout_ref(self.ref) self.ref = self.get_commit_hash() self.req.revision = self.ref + yield vcsrepo + self._repo = vcsrepo @classmethod def from_pipfile(cls, name, pipfile): @@ -627,12 +642,17 @@ def from_pipfile(cls, name, pipfile): "{0}+{1}".format(key, pipfile.get(key)) ).split("+", 1)[1] url_keys = [pipfile.get(key), composed_uri] - is_url = any(validity_fn(url_key) for url_key in url_keys for validity_fn in [is_valid_url, is_file_url]) - target_key = "uri" if is_url else "path" - creation_args[target_key] = pipfile.get(key) + if any(is_valid_url(k) for k in url_keys) or any(is_file_url(k) for k in url_keys): + creation_args["uri"] = pipfile.get(key) + else: + creation_args["path"] = pipfile.get(key) + if os.path.isabs(pipfile.get(key)): + creation_args["uri"] = Path(pipfile.get(key)).absolute().as_uri() else: creation_args[key] = pipfile.get(key) creation_args["name"] = name + print("Creating req from pipfile: %s" % pipfile) + print("Using creation args: %s" % creation_args) return cls(**creation_args) @classmethod @@ -669,7 +689,13 @@ def from_line(cls, line, editable=None, extras=None): @property def line_part(self): """requirements.txt compatible line part sans-extras""" - if self.req: + if self.is_local: + base_link = self.link + if not self.link: + base_link = self.get_link() + final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) if base_link.egg_fragment else "{0}" + base = final_format.format(self.vcs_uri) + elif self.req: base = self.req.line if base and self.extras and not extras_to_string(self.extras) in base: if self.subdirectory: @@ -694,7 +720,7 @@ def _choose_vcs_source(pipfile): @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=lambda k, v: v is not None and k.name != '_repo').copy() + pipfile_dict = attr.asdict(self, filter=lambda k, v: bool(v) is True and k.name != '_repo').copy() if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) name, _ = _strip_extras(pipfile_dict.pop("name")) @@ -752,7 +778,10 @@ def extras_as_pip(self): def commit_hash(self): if not self.is_vcs: return None - return self.req.get_commit_hash() + commit_hash = None + with self.req.locked_vcs_repo() as repo: + commit_hash = repo.get_commit_hash() + return commit_hash @specifiers.default def get_specifiers(self): diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index 3bfb05d5fd..5d3ec08fc9 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -23,34 +23,43 @@ def get_repo_instance(self): backend = VCS_SUPPORT._registry.get(self.vcs_type) return backend(url=self.url) + @property + def is_local(self): + url = self.url + if '+' in url: + url = url.split('+')[1] + return url.startswith("file") + def obtain(self): if not os.path.exists(self.checkout_directory): self.repo_instance.obtain(self.checkout_directory) if self.ref: - with vistir.contextmanagers.cd(self.checkout_directory): - self.update(self.ref) - self.commit_sha = self.get_commit_hash() + self.checkout_ref(self.ref) + self.commit_sha = self.get_commit_hash(self.ref) else: if not self.commit_sha: self.commit_sha = self.get_commit_hash() def checkout_ref(self, ref): - target_rev = self.repo_instance.make_rev_options(ref) if not self.repo_instance.is_commit_id_equal( self.checkout_directory, self.get_commit_hash(ref) ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref): - self.repo_instance.update(self.checkout_directory, self.url, target_rev) - self.commit_hash = self.get_commit_hash() + if not self.is_local: + self.update(ref) def update(self, ref): - target_rev = self.repo_instance.make_rev_options(ref) + target_ref = self.repo_instance.make_rev_options(ref) + sha = self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) + target_rev = target_ref.make_new(sha) if parse_version(pip_version) > parse_version("18.0"): self.repo_instance.update(self.checkout_directory, self.url, target_rev) else: - self.repo_instance.update(self.checkout_directory, target_rev) - self.commit_hash = self.get_commit_hash() + self.repo_instance.update(self.checkout_directory, target_ref) + self.commit_hash = self.get_commit_hash(ref) def get_commit_hash(self, ref=None): if ref: - return self.repo_instance.get_revision(self.checkout_directory) + target_ref = self.repo_instance.make_rev_options(ref) + return self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) + # return self.repo_instance.get_revision(self.checkout_directory) return self.repo_instance.get_revision(self.checkout_directory) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 6b937337ed..6dff468ac8 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -717,7 +717,7 @@ index 7e8cdf3..0a0d27d 100644 @@ -1,30 +1,42 @@ # -*- coding=utf-8 -*- import importlib - + -def do_import(module_path, subimport=None, old_path=None): + +def do_import(module_path, subimport=None, old_path=None, vendored_name=None): @@ -744,8 +744,8 @@ index 7e8cdf3..0a0d27d 100644 if subimport: return getattr(_tmp, subimport, _tmp) return _tmp -- - +- + -InstallRequirement = do_import('req.req_install', 'InstallRequirement') -parse_requirements = do_import('req.req_file', 'parse_requirements') -RequirementSet = do_import('req.req_set', 'RequirementSet') diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index c51db6ada2..c5915607c3 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -97,16 +97,18 @@ def test_file_urls_work(PipenvInstance, pip_src_dir): @pytest.mark.files @pytest.mark.urls @pytest.mark.needs_internet -def test_local_vcs_urls_work(PipenvInstance, pypi): +def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): + six_dir = tmpdir.join("six") + six_path = Path(six_dir.strpath) with PipenvInstance(pypi=pypi, chdir=True) as p: - six_path = Path(p.path).joinpath("six").absolute() c = delegator.run( - "git clone " "https://github.com/benjaminp/six.git {0}".format(six_path) + "git clone https://github.com/benjaminp/six.git {0}".format(six_dir.strpath) ) assert c.return_code == 0 c = p.pipenv("install git+{0}#egg=six".format(six_path.as_uri())) assert c.return_code == 0 + assert "six" in p.pipfile["packages"] @pytest.mark.e From 6f7dbe9e0839bc8de2089a0a28909fb57afa2261 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 4 Sep 2018 00:45:53 -0400 Subject: [PATCH 20/21] Windows edge case and news Signed-off-by: Dan Ryan --- news/1690.bugfix | 1 + news/2173.bugfix | 1 + news/2279.bugfix | 1 + news/2494.bugfix | 1 + news/2714.bugfix | 1 + news/2748.bugfix | 1 + news/2760.bugfix | 1 + news/2766.bugfix | 1 + news/2814.feature | 1 + .../requirementslib/models/requirements.py | 19 ++++++++----------- 10 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 news/1690.bugfix create mode 100644 news/2173.bugfix create mode 100644 news/2279.bugfix create mode 100644 news/2494.bugfix create mode 100644 news/2714.bugfix create mode 100644 news/2748.bugfix create mode 100644 news/2760.bugfix create mode 100644 news/2766.bugfix create mode 100644 news/2814.feature diff --git a/news/1690.bugfix b/news/1690.bugfix new file mode 100644 index 0000000000..ec94c663a4 --- /dev/null +++ b/news/1690.bugfix @@ -0,0 +1 @@ +VCS Refs for locked local editable dependencies will now update appropriately to the latest hash when running ``pipenv update``. diff --git a/news/2173.bugfix b/news/2173.bugfix new file mode 100644 index 0000000000..8d79613a4a --- /dev/null +++ b/news/2173.bugfix @@ -0,0 +1 @@ +``.tar.gz`` and ``.zip`` artifacts will now have dependencies installed even when they are missing from the lockfile. diff --git a/news/2279.bugfix b/news/2279.bugfix new file mode 100644 index 0000000000..f211ba4741 --- /dev/null +++ b/news/2279.bugfix @@ -0,0 +1 @@ +The command line parser will now handle multiple ``-e/--editable`` dependencies properly via click's option parser to help mitigate future parsing issues. diff --git a/news/2494.bugfix b/news/2494.bugfix new file mode 100644 index 0000000000..8aab779b41 --- /dev/null +++ b/news/2494.bugfix @@ -0,0 +1 @@ +Fixed a bug which could cause ``-i/--index`` arguments to sometimes be incorrectly picked up in packages. This is now handled in the command line parser. diff --git a/news/2714.bugfix b/news/2714.bugfix new file mode 100644 index 0000000000..74c7fac016 --- /dev/null +++ b/news/2714.bugfix @@ -0,0 +1 @@ +Fixed a bug which could cause the ``-e/--editable`` argument on a dependency to be accidentally parsed as a dependency itself. diff --git a/news/2748.bugfix b/news/2748.bugfix new file mode 100644 index 0000000000..fc807047a6 --- /dev/null +++ b/news/2748.bugfix @@ -0,0 +1 @@ +All markers are now included in ``pipenv lock --requirements`` output. diff --git a/news/2760.bugfix b/news/2760.bugfix new file mode 100644 index 0000000000..11bc2e639d --- /dev/null +++ b/news/2760.bugfix @@ -0,0 +1 @@ +Fixed a bug in marker resolution which could cause duplicate and non-deterministic markers. diff --git a/news/2766.bugfix b/news/2766.bugfix new file mode 100644 index 0000000000..f83cd0841e --- /dev/null +++ b/news/2766.bugfix @@ -0,0 +1 @@ +Fixed a bug in the dependency resolver which caused regular issues when handling ``setup.py`` based dependency resolution. diff --git a/news/2814.feature b/news/2814.feature new file mode 100644 index 0000000000..01818bc214 --- /dev/null +++ b/news/2814.feature @@ -0,0 +1 @@ +Deduplicate and refactor CLI to use stateful arguments and object passing. See `this issue `_ for reference. diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 471ebcea14..13a3a60ec9 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -638,21 +638,18 @@ def from_pipfile(cls, name, pipfile): pipfile[key] = sorted(dedup([extra.lower() for extra in extras])) if key in VCS_LIST: creation_args["vcs"] = key - composed_uri = add_ssh_scheme_to_git_uri( - "{0}+{1}".format(key, pipfile.get(key)) - ).split("+", 1)[1] - url_keys = [pipfile.get(key), composed_uri] - if any(is_valid_url(k) for k in url_keys) or any(is_file_url(k) for k in url_keys): - creation_args["uri"] = pipfile.get(key) + target = pipfile.get(key) + drive, path = os.path.splitdrive(target) + if not drive and not os.path.exists(target) and (is_valid_url(target) or + is_file_url(target) or target.startswith('git@')): + creation_args["uri"] = target else: - creation_args["path"] = pipfile.get(key) - if os.path.isabs(pipfile.get(key)): - creation_args["uri"] = Path(pipfile.get(key)).absolute().as_uri() + creation_args["path"] = target + if os.path.isabs(target): + creation_args["uri"] = path_to_url(target) else: creation_args[key] = pipfile.get(key) creation_args["name"] = name - print("Creating req from pipfile: %s" % pipfile) - print("Using creation args: %s" % creation_args) return cls(**creation_args) @classmethod From b2110f90ed8b6e2df5c5b51c970b72cefc8a9ed6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 4 Sep 2018 11:51:57 -0400 Subject: [PATCH 21/21] Fix option in cli and get released requirementslib Signed-off-by: Dan Ryan --- pipenv/cli/options.py | 5 +---- pipenv/vendor/requirementslib/__init__.py | 2 +- pipenv/vendor/vendor.txt | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 2d073288ce..cf9daa190d 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -93,10 +93,7 @@ def callback(ctx, param, value): def extra_index_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) - if isinstance(value, (tuple, list)): - state.extra_index_urls.extend(list(value)) - else: - state.extra_index_urls.append(value) + state.extra_index_urls.extend(list(value)) return value return option("--extra-index-url", multiple=True, expose_value=False, help=u"URLs to the extra PyPI compatible indexes to query for package lookups.", diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 8eb90d547e..0faea40b4e 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.1.6.dev0' +__version__ = '1.1.7.dev0' from .exceptions import RequirementError diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 9b6d7bea8a..d1667eeb00 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -27,7 +27,7 @@ requests==2.19.1 idna==2.7 urllib3==1.23 certifi==2018.8.24 -requirementslib==1.1.5 +requirementslib==1.1.6 attrs==18.1.0 distlib==0.2.7 packaging==17.1