diff --git a/MANIFEST.in b/MANIFEST.in index f593cf781c..e5cc05f2b1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,6 +24,8 @@ recursive-include docs/_templates *.html recursive-include docs/_static *.js *.css *.png recursive-exclude docs requirements*.txt + +prune peeps prune .buildkite prune .github prune .vsts-ci diff --git a/news/1690.bugfix b/news/1690.bugfix new file mode 100644 index 0000000000..ec94c663a4 --- /dev/null +++ b/news/1690.bugfix @@ -0,0 +1 @@ +VCS Refs for locked local editable dependencies will now update appropriately to the latest hash when running ``pipenv update``. diff --git a/news/2173.bugfix b/news/2173.bugfix new file mode 100644 index 0000000000..8d79613a4a --- /dev/null +++ b/news/2173.bugfix @@ -0,0 +1 @@ +``.tar.gz`` and ``.zip`` artifacts will now have dependencies installed even when they are missing from the lockfile. diff --git a/news/2279.bugfix b/news/2279.bugfix new file mode 100644 index 0000000000..f211ba4741 --- /dev/null +++ b/news/2279.bugfix @@ -0,0 +1 @@ +The command line parser will now handle multiple ``-e/--editable`` dependencies properly via click's option parser to help mitigate future parsing issues. diff --git a/news/2494.bugfix b/news/2494.bugfix new file mode 100644 index 0000000000..8aab779b41 --- /dev/null +++ b/news/2494.bugfix @@ -0,0 +1 @@ +Fixed a bug which could cause ``-i/--index`` arguments to sometimes be incorrectly picked up in packages. This is now handled in the command line parser. diff --git a/news/2714.bugfix b/news/2714.bugfix new file mode 100644 index 0000000000..74c7fac016 --- /dev/null +++ b/news/2714.bugfix @@ -0,0 +1 @@ +Fixed a bug which could cause the ``-e/--editable`` argument on a dependency to be accidentally parsed as a dependency itself. diff --git a/news/2748.bugfix b/news/2748.bugfix new file mode 100644 index 0000000000..fc807047a6 --- /dev/null +++ b/news/2748.bugfix @@ -0,0 +1 @@ +All markers are now included in ``pipenv lock --requirements`` output. diff --git a/news/2760.bugfix b/news/2760.bugfix new file mode 100644 index 0000000000..11bc2e639d --- /dev/null +++ b/news/2760.bugfix @@ -0,0 +1 @@ +Fixed a bug in marker resolution which could cause duplicate and non-deterministic markers. diff --git a/news/2766.bugfix b/news/2766.bugfix new file mode 100644 index 0000000000..f83cd0841e --- /dev/null +++ b/news/2766.bugfix @@ -0,0 +1 @@ +Fixed a bug in the dependency resolver which caused regular issues when handling ``setup.py`` based dependency resolution. diff --git a/news/2814.feature b/news/2814.feature new file mode 100644 index 0000000000..01818bc214 --- /dev/null +++ b/news/2814.feature @@ -0,0 +1 @@ +Deduplicate and refactor CLI to use stateful arguments and object passing. See `this issue `_ for reference. diff --git a/pipenv/_compat.py b/pipenv/_compat.py index b1861e4d21..558df3b846 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -90,7 +90,7 @@ class TemporaryDirectory(object): in it are removed. """ - def __init__(self, suffix=None, prefix=None, dir=None): + def __init__(self, suffix="", prefix="", dir=None): if "RAM_DISK" in os.environ: import uuid diff --git a/pipenv/cli.py b/pipenv/cli.py deleted file mode 100644 index 908c6f998c..0000000000 --- a/pipenv/cli.py +++ /dev/null @@ -1,1077 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import sys -from click import ( - argument, - command, - echo, - edit, - group, - Group, - option, - pass_context, - Option, - version_option, - BadParameter, -) -from click_didyoumean import DYMCommandCollection - -import click_completion -import crayons -import delegator - -from .__version__ import __version__ - -from . import environments -from .utils import is_valid_url - -# Enable shell completion. -click_completion.init() -CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) - - -class PipenvGroup(Group): - """Custom Group class provides formatted main help""" - - def get_help_option(self, ctx): - from .core import format_help - - """Override for showing formatted main help via --help and -h options""" - help_options = self.get_help_option_names(ctx) - if not help_options or not self.add_help_option: - return - - def show_help(ctx, param, value): - if value and not ctx.resilient_parsing: - if not ctx.invoked_subcommand: - # legit main help - echo(format_help(ctx.get_help())) - else: - # legit sub-command help - echo(ctx.get_help(), color=ctx.color) - ctx.exit() - - return Option( - help_options, - is_flag=True, - is_eager=True, - expose_value=False, - callback=show_help, - help="Show this message and exit.", - ) - - -def setup_verbosity(ctx, param, value): - if not value: - return - import logging - logging.getLogger("pip").setLevel(logging.INFO) - environments.PIPENV_VERBOSITY = 1 - - -def validate_python_path(ctx, param, value): - # Validating the Python path is complicated by accepting a number of - # friendly options: the default will be boolean False to enable - # autodetection but it may also be a value which will be searched in - # the path or an absolute path. To report errors as early as possible - # we'll report absolute paths which do not exist: - if isinstance(value, (str, bytes)): - if os.path.isabs(value) and not os.path.isfile(value): - raise BadParameter("Expected Python at path %s does not exist" % value) - return value - - -def validate_pypi_mirror(ctx, param, value): - if value and not is_valid_url(value): - raise BadParameter("Invalid PyPI mirror URL: %s" % value) - return value - - -@group(cls=PipenvGroup, invoke_without_command=True, context_settings=CONTEXT_SETTINGS) -@option("--where", is_flag=True, default=False, help="Output project home information.") -@option("--venv", is_flag=True, default=False, help="Output virtualenv information.") -@option( - "--py", is_flag=True, default=False, help="Output Python interpreter information." -) -@option( - "--envs", is_flag=True, default=False, help="Output Environment Variable options." -) -@option("--rm", is_flag=True, default=False, help="Remove the virtualenv.") -@option("--bare", is_flag=True, default=False, help="Minimal output.") -@option( - "--completion", - is_flag=True, - default=False, - help="Output completion (to be eval'd).", -) -@option("--man", is_flag=True, default=False, help="Display manpage.") -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--site-packages", - is_flag=True, - default=False, - help="Enable site-packages for the virtualenv.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -@option( - "--support", - is_flag=True, - help="Output diagnostic information for use in GitHub issues.", -) -@option("--clear", is_flag=True, help="Clears caches (pipenv, pip, and pip-tools).") -@version_option(prog_name=crayons.normal("pipenv", bold=True), version=__version__) -@pass_context -def cli( - ctx, - where=False, - venv=False, - rm=False, - bare=False, - three=False, - python=False, - help=False, - py=False, - site_packages=False, - envs=False, - man=False, - completion=False, - pypi_mirror=None, - support=None, - clear=False, -): - # Handle this ASAP to make shell startup fast. - if completion: - from . import shells - - try: - shell = shells.detect_info()[0] - except shells.ShellDetectionFailure: - echo( - "Fail to detect shell. Please provide the {0} environment " - "variable.".format(crayons.normal("PIPENV_SHELL", bold=True)), - err=True, - ) - sys.exit(1) - print(click_completion.get_code(shell=shell, prog_name="pipenv")) - sys.exit(0) - - from .core import ( - system_which, - do_py, - warn_in_virtualenv, - do_where, - project, - spinner, - cleanup_virtualenv, - ensure_project, - format_help, - do_clear, - ) - - if man: - if system_which("man"): - path = os.sep.join([os.path.dirname(__file__), "pipenv.1"]) - os.execle(system_which("man"), "man", path, os.environ) - else: - echo("man does not appear to be available on your system.", err=True) - if envs: - echo("The following environment variables can be set, to do various things:\n") - for key in environments.__dict__: - if key.startswith("PIPENV"): - echo(" - {0}".format(crayons.normal(key, bold=True))) - echo( - "\nYou can learn more at:\n {0}".format( - crayons.green( - "http://docs.pipenv.org/advanced/#configuration-with-environment-variables" - ) - ) - ) - sys.exit(0) - warn_in_virtualenv() - if ctx.invoked_subcommand is None: - # --where was passed… - if where: - do_where(bare=True) - sys.exit(0) - elif py: - do_py() - sys.exit() - # --support was passed… - elif support: - from .help import get_pipenv_diagnostics - - get_pipenv_diagnostics() - sys.exit(0) - # --clear was passed… - elif clear: - do_clear() - sys.exit(0) - - # --venv was passed… - elif venv: - # There is no virtualenv yet. - if not project.virtualenv_exists: - echo( - crayons.red("No virtualenv has been created for this project yet!"), - err=True, - ) - sys.exit(1) - else: - echo(project.virtualenv_location) - sys.exit(0) - # --rm was passed… - elif rm: - # Abort if --system (or running in a virtualenv). - if environments.PIPENV_USE_SYSTEM: - echo( - crayons.red( - "You are attempting to remove a virtualenv that " - "Pipenv did not create. Aborting." - ) - ) - sys.exit(1) - if project.virtualenv_exists: - loc = project.virtualenv_location - echo( - crayons.normal( - u"{0} ({1})…".format( - crayons.normal("Removing virtualenv", bold=True), - crayons.green(loc), - ) - ) - ) - with spinner(): - # Remove the virtualenv. - cleanup_virtualenv(bare=True) - sys.exit(0) - else: - echo( - crayons.red( - "No virtualenv has been created for this project yet!", - bold=True, - ), - err=True, - ) - sys.exit(1) - # --two / --three was passed… - if (python or three is not None) or site_packages: - ensure_project( - three=three, - python=python, - warn=True, - site_packages=site_packages, - pypi_mirror=pypi_mirror, - clear=clear, - ) - # Check this again before exiting for empty ``pipenv`` command. - elif ctx.invoked_subcommand is None: - # Display help to user, if no commands were passed. - echo(format_help(ctx.get_help())) - - -@command( - short_help="Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.", - context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), -) -@argument("package_name", default=False) -@argument("more_packages", nargs=-1) -@option( - "--dev", - "-d", - is_flag=True, - default=False, - help="Install package(s) in [dev-packages].", -) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -@option("--system", is_flag=True, default=False, help="System pip management.") -@option( - "--requirements", - "-r", - nargs=1, - default=False, - help="Import a requirements.txt file.", -) -@option("--code", "-c", nargs=1, default=False, help="Import from codebase.") -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - callback=setup_verbosity, - help="Verbose mode.", -) -@option( - "--ignore-pipfile", - is_flag=True, - default=False, - help="Ignore Pipfile when installing, using the Pipfile.lock.", -) -@option( - "--sequential", - is_flag=True, - default=False, - help="Install dependencies one-at-a-time, instead of concurrently.", -) -@option( - "--skip-lock", - is_flag=True, - default=False, - help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", -) -@option( - "--deploy", - is_flag=True, - default=False, - help=u"Abort if the Pipfile.lock is out-of-date, or Python version is wrong.", -) -@option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.") -@option( - "--keep-outdated", - is_flag=True, - default=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", -) -@option( - "--selective-upgrade", - is_flag=True, - default=False, - help="Update specified packages.", -) -def install( - package_name=False, - more_packages=False, - dev=False, - three=False, - python=False, - pypi_mirror=None, - system=False, - lock=True, - ignore_pipfile=False, - skip_lock=False, - requirements=False, - sequential=False, - pre=False, - code=False, - deploy=False, - keep_outdated=False, - selective_upgrade=False, -): - """Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.""" - from .core import do_install - - do_install( - package_name=package_name, - more_packages=more_packages, - dev=dev, - three=three, - python=python, - pypi_mirror=pypi_mirror, - system=system, - lock=lock, - ignore_pipfile=ignore_pipfile, - skip_lock=skip_lock, - requirements=requirements, - sequential=sequential, - pre=pre, - code=code, - deploy=deploy, - keep_outdated=keep_outdated, - selective_upgrade=selective_upgrade, - ) - - -@command(short_help="Un-installs a provided package and removes it from Pipfile.") -@argument("package_name", default=False) -@argument("more_packages", nargs=-1) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option("--system", is_flag=True, default=False, help="System pip management.") -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - help="Verbose mode.", - callback=setup_verbosity, -) -@option("--lock", is_flag=True, default=True, help="Lock afterwards.") -@option( - "--all-dev", - is_flag=True, - default=False, - help="Un-install all package from [dev-packages].", -) -@option( - "--all", - is_flag=True, - default=False, - help="Purge all package(s) from virtualenv. Does not edit Pipfile.", -) -@option( - "--keep-outdated", - is_flag=True, - default=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -def uninstall( - package_name=False, - more_packages=False, - three=None, - python=False, - system=False, - lock=False, - all_dev=False, - all=False, - keep_outdated=False, - pypi_mirror=None, -): - """Un-installs a provided package and removes it from Pipfile.""" - from .core import do_uninstall - - do_uninstall( - package_name=package_name, - more_packages=more_packages, - three=three, - python=python, - system=system, - lock=lock, - all_dev=all_dev, - all=all, - keep_outdated=keep_outdated, - pypi_mirror=pypi_mirror, - ) - - -@command(short_help="Generates Pipfile.lock.") -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - help="Verbose mode.", - callback=setup_verbosity, -) -@option( - "--requirements", - "-r", - is_flag=True, - default=False, - help="Generate output compatible with requirements.txt.", -) -@option( - "--dev", - "-d", - is_flag=True, - default=False, - help="Generate output compatible with requirements.txt for the development dependencies.", -) -@option("--clear", is_flag=True, default=False, help="Clear the dependency cache.") -@option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.") -@option( - "--keep-outdated", - is_flag=True, - default=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", -) -def lock( - three=None, - python=False, - pypi_mirror=None, - requirements=False, - dev=False, - clear=False, - pre=False, - keep_outdated=False, -): - """Generates Pipfile.lock.""" - from .core import ensure_project, do_init, do_lock - - # Ensure that virtualenv is available. - ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) - if requirements: - do_init(dev=dev, requirements=requirements, pypi_mirror=pypi_mirror) - do_lock( - clear=clear, - pre=pre, - keep_outdated=keep_outdated, - pypi_mirror=pypi_mirror, - ) - - -@command( - short_help="Spawns a shell within the virtualenv.", - context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), -) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--fancy", - is_flag=True, - default=False, - help="Run in shell in fancy mode (for elegantly configured shells).", -) -@option( - "--anyway", - is_flag=True, - default=False, - help="Always spawn a subshell, even if one is already spawned.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -@argument("shell_args", nargs=-1) -def shell( - three=None, - python=False, - fancy=False, - shell_args=None, - anyway=False, - pypi_mirror=None, -): - """Spawns a shell within the virtualenv.""" - from .core import load_dot_env, do_shell - - # Prevent user from activating nested environments. - if "PIPENV_ACTIVE" in os.environ: - # If PIPENV_ACTIVE is set, VIRTUAL_ENV should always be set too. - venv_name = os.environ.get("VIRTUAL_ENV", "UNKNOWN_VIRTUAL_ENVIRONMENT") - if not anyway: - echo( - "{0} {1} {2}\nNo action taken to avoid nested environments.".format( - crayons.normal("Shell for"), - crayons.green(venv_name, bold=True), - crayons.normal("already activated.", bold=True), - ), - err=True, - ) - sys.exit(1) - # Load .env file. - load_dot_env() - # Use fancy mode for Windows. - if os.name == "nt": - fancy = True - do_shell( - three=three, - python=python, - fancy=fancy, - shell_args=shell_args, - pypi_mirror=pypi_mirror, - ) - - -@command( - add_help_option=False, - short_help="Spawns a command installed into the virtualenv.", - context_settings=dict( - ignore_unknown_options=True, - allow_interspersed_args=False, - allow_extra_args=True, - ), -) -@argument("command") -@argument("args", nargs=-1) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -def run(command, args, three=None, python=False, pypi_mirror=None): - """Spawns a command installed into the virtualenv.""" - from .core import do_run - - do_run( - command=command, args=args, three=three, python=python, pypi_mirror=pypi_mirror - ) - - -@command( - short_help="Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.", - context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), -) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option("--system", is_flag=True, default=False, help="Use system Python.") -@option( - "--unused", - nargs=1, - default=False, - help="Given a code path, show potentially unused dependencies.", -) -@option( - "--ignore", - "-i", - multiple=True, - help="Ignore specified vulnerability during safety checks.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -@argument("args", nargs=-1) -def check( - three=None, - python=False, - system=False, - unused=False, - style=False, - ignore=None, - args=None, - pypi_mirror=None, -): - """Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.""" - from .core import do_check - - do_check( - three=three, - python=python, - system=system, - unused=unused, - ignore=ignore, - args=args, - pypi_mirror=pypi_mirror, - ) - - -@command(short_help="Runs lock, then sync.") -@argument("more_packages", nargs=-1) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - help="Verbose mode.", - callback=setup_verbosity, -) -@option( - "--dev", - "-d", - is_flag=True, - default=False, - help="Install package(s) in [dev-packages].", -) -@option("--clear", is_flag=True, default=False, help="Clear the dependency cache.") -@option("--bare", is_flag=True, default=False, help="Minimal output.") -@option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.") -@option( - "--keep-outdated", - is_flag=True, - default=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", -) -@option( - "--sequential", - is_flag=True, - default=False, - help="Install dependencies one-at-a-time, instead of concurrently.", -) -@option( - "--outdated", is_flag=True, default=False, help=u"List out-of-date dependencies." -) -@option("--dry-run", is_flag=True, default=None, help=u"List out-of-date dependencies.") -@argument("package", default=False) -@pass_context -def update( - ctx, - three=None, - python=False, - pypi_mirror=None, - system=False, - clear=False, - keep_outdated=False, - pre=False, - dev=False, - bare=False, - sequential=False, - package=None, - dry_run=None, - outdated=False, - more_packages=None, -): - """Runs lock, then sync.""" - from .core import ( - ensure_project, - do_outdated, - do_lock, - do_sync, - project, - ) - - ensure_project(three=three, python=python, warn=True, pypi_mirror=pypi_mirror) - if not outdated: - outdated = bool(dry_run) - if outdated: - do_outdated(pypi_mirror=pypi_mirror) - if not package: - echo( - "{0} {1} {2} {3}{4}".format( - crayons.white("Running", bold=True), - crayons.red("$ pipenv lock", bold=True), - crayons.white("then", bold=True), - crayons.red("$ pipenv sync", bold=True), - crayons.white(".", bold=True), - ) - ) - else: - for package in [package] + list(more_packages) or []: - if package not in project.all_packages: - echo( - "{0}: {1} was not found in your Pipfile! Aborting." - "".format( - crayons.red("Warning", bold=True), - crayons.green(package, bold=True), - ), - err=True, - ) - sys.exit(1) - - do_lock( - clear=clear, - pre=pre, - keep_outdated=keep_outdated, - pypi_mirror=pypi_mirror, - ) - do_sync( - ctx=ctx, - dev=dev, - three=three, - python=python, - bare=bare, - dont_upgrade=False, - user=False, - clear=clear, - unused=False, - sequential=sequential, - pypi_mirror=pypi_mirror, - ) - - -@command(short_help=u"Displays currently-installed dependency graph information.") -@option("--bare", is_flag=True, default=False, help="Minimal output.") -@option("--json", is_flag=True, default=False, help="Output JSON.") -@option("--json-tree", is_flag=True, default=False, help="Output JSON in nested tree.") -@option("--reverse", is_flag=True, default=False, help="Reversed dependency graph.") -def graph(bare=False, json=False, json_tree=False, reverse=False): - """Displays currently-installed dependency graph information.""" - from .core import do_graph - - do_graph(bare=bare, json=json, json_tree=json_tree, reverse=reverse) - - -@command(short_help="View a given module in your editor.", name="open") -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -@argument("module", nargs=1) -def run_open(module, three=None, python=None, pypi_mirror=None): - """View a given module in your editor.""" - from .core import which, ensure_project - - # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) - c = delegator.run( - '{0} -c "import {1}; print({1}.__file__);"'.format(which("python"), module) - ) - try: - assert c.return_code == 0 - except AssertionError: - echo(crayons.red("Module not found!")) - sys.exit(1) - if "__init__.py" in c.out: - p = os.path.dirname(c.out.strip().rstrip("cdo")) - else: - p = c.out.strip().rstrip("cdo") - echo(crayons.normal("Opening {0!r} in your EDITOR.".format(p), bold=True)) - edit(filename=p) - sys.exit(0) - - -@command(short_help="Installs all packages specified in Pipfile.lock.") -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - help="Verbose mode.", - callback=setup_verbosity, -) -@option( - "--dev", - "-d", - is_flag=True, - default=False, - help="Additionally install package(s) in [dev-packages].", -) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--pypi-mirror", - default=environments.PIPENV_PYPI_MIRROR, - nargs=1, - callback=validate_pypi_mirror, - help="Specify a PyPI mirror.", -) -@option("--bare", is_flag=True, default=False, help="Minimal output.") -@option("--clear", is_flag=True, default=False, help="Clear the dependency cache.") -@option( - "--sequential", - is_flag=True, - default=False, - help="Install dependencies one-at-a-time, instead of concurrently.", -) -@pass_context -def sync( - ctx, - dev=False, - three=None, - python=None, - bare=False, - dont_upgrade=False, - user=False, - clear=False, - unused=False, - package_name=None, - sequential=False, - pypi_mirror=None, -): - """Installs all packages specified in Pipfile.lock.""" - from .core import do_sync - - do_sync( - ctx=ctx, - dev=dev, - three=three, - python=python, - bare=bare, - dont_upgrade=dont_upgrade, - user=user, - clear=clear, - unused=unused, - sequential=sequential, - pypi_mirror=pypi_mirror, - ) - - -@command(short_help="Uninstalls all packages not specified in Pipfile.lock.") -@option( - "--verbose", - "-v", - is_flag=True, - expose_value=False, - help="Verbose mode.", - callback=setup_verbosity, -) -@option( - "--three/--two", - is_flag=True, - default=None, - help="Use Python 3/2 when creating virtualenv.", -) -@option( - "--python", - default=False, - nargs=1, - callback=validate_python_path, - help="Specify which version of Python virtualenv should use.", -) -@option( - "--dry-run", - is_flag=True, - default=False, - help="Just output unneeded packages.", -) -@pass_context -def clean(ctx, three=None, python=None, dry_run=False, bare=False, user=False): - """Uninstalls all packages not specified in Pipfile.lock.""" - from .core import do_clean - - do_clean( - ctx=ctx, - three=three, python=python, - dry_run=dry_run, - ) - - -# Install click commands. -cli.add_command(graph) -cli.add_command(install) -cli.add_command(uninstall) -cli.add_command(sync) -cli.add_command(lock) -cli.add_command(check) -cli.add_command(clean) -cli.add_command(shell) -cli.add_command(run) -cli.add_command(update) -cli.add_command(run_open) -# Only invoke the "did you mean" when an argument wasn't passed (it breaks those). -if "-" not in "".join(sys.argv) and len(sys.argv) > 1: - cli = DYMCommandCollection(sources=[cli]) -if __name__ == "__main__": - cli() diff --git a/pipenv/cli/__init__.py b/pipenv/cli/__init__.py new file mode 100644 index 0000000000..605f4c10ec --- /dev/null +++ b/pipenv/cli/__init__.py @@ -0,0 +1,3 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import +from .command import cli diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py new file mode 100644 index 0000000000..f6c0d13385 --- /dev/null +++ b/pipenv/cli/command.py @@ -0,0 +1,609 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +import os +import sys + +import crayons +import delegator + +from click import ( + argument, echo, edit, group, option, pass_context, secho, version_option +) + +import click_completion + +from click_didyoumean import DYMCommandCollection + +from .. import environments +from ..__version__ import __version__ +from .options import ( + CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, + general_options, install_options, lock_options, pass_state, skip_lock_option, + pypi_mirror_option, python_option, requirementstxt_option, sync_options, + system_option, three_option, verbose_option, uninstall_options +) + + +# Enable shell completion. +click_completion.init() + + +@group(cls=PipenvGroup, invoke_without_command=True, context_settings=CONTEXT_SETTINGS) +@option("--where", is_flag=True, default=False, help="Output project home information.") +@option("--venv", is_flag=True, default=False, help="Output virtualenv information.") +@option( + "--py", is_flag=True, default=False, help="Output Python interpreter information." +) +@option( + "--envs", is_flag=True, default=False, help="Output Environment Variable options." +) +@option("--rm", is_flag=True, default=False, help="Remove the virtualenv.") +@option("--bare", is_flag=True, default=False, help="Minimal output.") +@option( + "--completion", + is_flag=True, + default=False, + help="Output completion (to be eval'd).", +) +@option("--man", is_flag=True, default=False, help="Display manpage.") +@option( + "--support", + is_flag=True, + help="Output diagnostic information for use in GitHub issues.", +) +@general_options +@version_option(prog_name=crayons.normal("pipenv", bold=True), version=__version__) +@pass_state +@pass_context +def cli( + ctx, + state, + where=False, + venv=False, + rm=False, + bare=False, + three=False, + python=False, + help=False, + py=False, + site_packages=False, + envs=False, + man=False, + completion=False, + pypi_mirror=None, + support=None, + clear=False, + **kwargs +): + # Handle this ASAP to make shell startup fast. + if completion: + from .. import shells + + try: + shell = shells.detect_info()[0] + except shells.ShellDetectionFailure: + echo( + "Fail to detect shell. Please provide the {0} environment " + "variable.".format(crayons.normal("PIPENV_SHELL", bold=True)), + err=True, + ) + ctx.abort() + print(click_completion.get_code(shell=shell, prog_name="pipenv")) + return 0 + + from ..core import ( + system_which, + do_py, + warn_in_virtualenv, + do_where, + project, + spinner, + cleanup_virtualenv, + ensure_project, + format_help, + do_clear, + ) + + if man: + if system_which("man"): + path = os.sep.join([os.path.dirname(__file__), "pipenv.1"]) + os.execle(system_which("man"), "man", path, os.environ) + return 0 + else: + secho("man does not appear to be available on your system.", fg="yellow", bold=True, err=True) + return 1 + if envs: + echo("The following environment variables can be set, to do various things:\n") + for key in environments.__dict__: + if key.startswith("PIPENV"): + echo(" - {0}".format(crayons.normal(key, bold=True))) + echo( + "\nYou can learn more at:\n {0}".format( + crayons.green( + "http://docs.pipenv.org/advanced/#configuration-with-environment-variables" + ) + ) + ) + return 0 + warn_in_virtualenv() + if ctx.invoked_subcommand is None: + # --where was passed… + if where: + do_where(bare=True) + return 0 + elif py: + do_py() + return 0 + # --support was passed… + elif support: + from ..help import get_pipenv_diagnostics + + get_pipenv_diagnostics() + return 0 + # --clear was passed… + elif clear: + do_clear() + return 0 + + # --venv was passed… + elif venv: + # There is no virtualenv yet. + if not project.virtualenv_exists: + echo( + crayons.red("No virtualenv has been created for this project yet!"), + err=True, + ) + ctx.abort() + else: + echo(project.virtualenv_location) + return 0 + # --rm was passed… + elif rm: + # Abort if --system (or running in a virtualenv). + if environments.PIPENV_USE_SYSTEM: + echo( + crayons.red( + "You are attempting to remove a virtualenv that " + "Pipenv did not create. Aborting." + ) + ) + ctx.abort() + if project.virtualenv_exists: + loc = project.virtualenv_location + echo( + crayons.normal( + u"{0} ({1})…".format( + crayons.normal("Removing virtualenv", bold=True), + crayons.green(loc), + ) + ) + ) + with spinner(): + # Remove the virtualenv. + cleanup_virtualenv(bare=True) + return 0 + else: + echo( + crayons.red( + "No virtualenv has been created for this project yet!", + bold=True, + ), + err=True, + ) + ctx.abort() + # --two / --three was passed… + if (state.python or state.three is not None) or site_packages: + ensure_project( + three=state.three, + python=state.python, + warn=True, + site_packages=state.site_packages, + pypi_mirror=state.pypi_mirror, + clear=state.clear, + ) + # Check this again before exiting for empty ``pipenv`` command. + elif ctx.invoked_subcommand is None: + # Display help to user, if no commands were passed. + echo(format_help(ctx.get_help())) + + +@cli.command( + short_help="Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.", + context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), +) +@system_option +@code_option +@deploy_option +@skip_lock_option +@install_options +@pass_state +@pass_context +def install( + ctx, + state, + **kwargs +): + """Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.""" + from ..core import do_install + + retcode = do_install( + dev=state.installstate.dev, + three=state.three, + python=state.python, + pypi_mirror=state.pypi_mirror, + system=state.system, + lock=not state.installstate.skip_lock, + ignore_pipfile=state.installstate.ignore_pipfile, + skip_lock=state.installstate.skip_lock, + requirements=state.installstate.requirementstxt, + sequential=state.installstate.sequential, + pre=state.installstate.pre, + code=state.installstate.code, + deploy=state.installstate.deploy, + keep_outdated=state.installstate.keep_outdated, + selective_upgrade=state.installstate.selective_upgrade, + index_url=state.index, + extra_index_url=state.extra_index_urls, + packages=state.installstate.packages, + editable_packages=state.installstate.editables, + ) + if retcode: + ctx.abort() + + +@cli.command(short_help="Un-installs a provided package and removes it from Pipfile.") +@option("--lock", is_flag=True, default=True, help="Lock afterwards.") +@option( + "--all-dev", + is_flag=True, + default=False, + help="Un-install all package from [dev-packages].", +) +@option( + "--all", + is_flag=True, + default=False, + help="Purge all package(s) from virtualenv. Does not edit Pipfile.", +) +@uninstall_options +@pass_state +def uninstall( + state, + lock=False, + all_dev=False, + all=False, + **kwargs +): + """Un-installs a provided package and removes it from Pipfile.""" + from ..core import do_uninstall + + retcode = do_uninstall( + packages=state.installstate.packages, + editable_packages=state.installstate.editables, + three=state.three, + python=state.python, + system=state.system, + lock=lock, + all_dev=all_dev, + all=all, + keep_outdated=state.installstate.keep_outdated, + pypi_mirror=state.pypi_mirror, + ) + if retcode: + sys.exit(retcode) + + +@cli.command(short_help="Generates Pipfile.lock.") +@lock_options +@pass_state +def lock( + state, + **kwargs +): + """Generates Pipfile.lock.""" + from ..core import ensure_project, do_init, do_lock + + # Ensure that virtualenv is available. + ensure_project(three=state.three, python=state.python, pypi_mirror=state.pypi_mirror) + if state.installstate.requirementstxt: + do_init(dev=state.installstate.dev, requirements=state.installstate.requirementstxt, + pypi_mirror=state.pypi_mirror) + do_lock( + clear=state.clear, + pre=state.installstate.pre, + keep_outdated=state.installstate.keep_outdated, + pypi_mirror=state.pypi_mirror, + ) + + +@cli.command( + short_help="Spawns a shell within the virtualenv.", + context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), +) +@option( + "--fancy", + is_flag=True, + default=False, + help="Run in shell in fancy mode (for elegantly configured shells).", +) +@option( + "--anyway", + is_flag=True, + default=False, + help="Always spawn a subshell, even if one is already spawned.", +) +@argument("shell_args", nargs=-1) +@pypi_mirror_option +@three_option +@python_option +@pass_state +def shell( + state, + fancy=False, + shell_args=None, + anyway=False, +): + """Spawns a shell within the virtualenv.""" + from ..core import load_dot_env, do_shell + + # Prevent user from activating nested environments. + if "PIPENV_ACTIVE" in os.environ: + # If PIPENV_ACTIVE is set, VIRTUAL_ENV should always be set too. + venv_name = os.environ.get("VIRTUAL_ENV", "UNKNOWN_VIRTUAL_ENVIRONMENT") + if not anyway: + echo( + "{0} {1} {2}\nNo action taken to avoid nested environments.".format( + crayons.normal("Shell for"), + crayons.green(venv_name, bold=True), + crayons.normal("already activated.", bold=True), + ), + err=True, + ) + sys.exit(1) + # Load .env file. + load_dot_env() + # Use fancy mode for Windows. + if os.name == "nt": + fancy = True + do_shell( + three=state.three, + python=state.python, + fancy=fancy, + shell_args=shell_args, + pypi_mirror=state.pypi_mirror, + ) + + +@cli.command( + add_help_option=False, + short_help="Spawns a command installed into the virtualenv.", + context_settings=dict( + ignore_unknown_options=True, + allow_interspersed_args=False, + allow_extra_args=True, + ), +) +@common_options +@argument("command") +@argument("args", nargs=-1) +@pass_state +def run(state, command, args): + """Spawns a command installed into the virtualenv.""" + from ..core import do_run + + do_run( + command=command, args=args, three=state.three, python=state.python, pypi_mirror=state.pypi_mirror + ) + + +@cli.command( + short_help="Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.", + context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), +) +@option( + "--unused", + nargs=1, + default=False, + help="Given a code path, show potentially unused dependencies.", +) +@option( + "--ignore", + "-i", + multiple=True, + help="Ignore specified vulnerability during safety checks.", +) +@common_options +@system_option +@argument("args", nargs=-1) +@pass_state +def check( + state, + unused=False, + style=False, + ignore=None, + args=None, + **kwargs +): + """Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.""" + from ..core import do_check + + do_check( + three=state.three, + python=state.python, + system=state.system, + unused=unused, + ignore=ignore, + args=args, + pypi_mirror=state.pypi_mirror, + ) + + +@cli.command(short_help="Runs lock, then sync.") +@option("--bare", is_flag=True, default=False, help="Minimal output.") +@option( + "--outdated", is_flag=True, default=False, help=u"List out-of-date dependencies." +) +@option("--dry-run", is_flag=True, default=None, help=u"List out-of-date dependencies.") +@install_options +@pass_state +@pass_context +def update( + ctx, + state, + bare=False, + dry_run=None, + outdated=False, + **kwargs +): + """Runs lock, then sync.""" + from ..core import ( + ensure_project, + do_outdated, + do_lock, + do_sync, + project, + ) + + ensure_project(three=state.three, python=state.python, warn=True, pypi_mirror=state.pypi_mirror) + if not outdated: + outdated = bool(dry_run) + if outdated: + do_outdated(pypi_mirror=state.pypi_mirror) + packages = [p for p in state.installstate.packages if p] + editable = [p for p in state.installstate.editables if p] + if not packages: + echo( + "{0} {1} {2} {3}{4}".format( + crayons.white("Running", bold=True), + crayons.red("$ pipenv lock", bold=True), + crayons.white("then", bold=True), + crayons.red("$ pipenv sync", bold=True), + crayons.white(".", bold=True), + ) + ) + else: + for package in packages + editable: + if package not in project.all_packages: + echo( + "{0}: {1} was not found in your Pipfile! Aborting." + "".format( + crayons.red("Warning", bold=True), + crayons.green(package, bold=True), + ), + err=True, + ) + ctx.abort() + + do_lock( + clear=state.clear, + pre=state.installstate.pre, + keep_outdated=state.installstate.keep_outdated, + pypi_mirror=state.pypi_mirror, + ) + do_sync( + ctx=ctx, + dev=state.installstate.dev, + three=state.three, + python=state.python, + bare=bare, + dont_upgrade=not state.installstate.keep_outdated, + user=False, + clear=state.clear, + unused=False, + sequential=state.installstate.sequential, + pypi_mirror=state.pypi_mirror, + ) + + +@cli.command(short_help=u"Displays currently-installed dependency graph information.") +@option("--bare", is_flag=True, default=False, help="Minimal output.") +@option("--json", is_flag=True, default=False, help="Output JSON.") +@option("--json-tree", is_flag=True, default=False, help="Output JSON in nested tree.") +@option("--reverse", is_flag=True, default=False, help="Reversed dependency graph.") +def graph(bare=False, json=False, json_tree=False, reverse=False): + """Displays currently-installed dependency graph information.""" + from ..core import do_graph + + do_graph(bare=bare, json=json, json_tree=json_tree, reverse=reverse) + + +@cli.command(short_help="View a given module in your editor.", name="open") +@common_options +@argument("module", nargs=1) +@pass_state +def run_open(state, module, *args, **kwargs): + """View a given module in your editor.""" + from ..core import which, ensure_project + + # Ensure that virtualenv is available. + ensure_project(three=state.three, python=state.python, validate=False, + pypi_mirror=state.pypi_mirror) + c = delegator.run( + '{0} -c "import {1}; print({1}.__file__);"'.format(which("python"), module) + ) + try: + assert c.return_code == 0 + except AssertionError: + echo(crayons.red("Module not found!")) + sys.exit(1) + if "__init__.py" in c.out: + p = os.path.dirname(c.out.strip().rstrip("cdo")) + else: + p = c.out.strip().rstrip("cdo") + echo(crayons.normal("Opening {0!r} in your EDITOR.".format(p), bold=True)) + edit(filename=p) + return 0 + + +@cli.command(short_help="Installs all packages specified in Pipfile.lock.") +@option("--bare", is_flag=True, default=False, help="Minimal output.") +@sync_options +@pass_state +@pass_context +def sync( + ctx, + state, + bare=False, + user=False, + unused=False, + **kwargs +): + """Installs all packages specified in Pipfile.lock.""" + from ..core import do_sync + + retcode = do_sync( + ctx=ctx, + dev=state.installstate.dev, + three=state.three, + python=state.python, + bare=bare, + dont_upgrade=(not state.installstate.keep_outdated), + user=user, + clear=state.clear, + unused=unused, + sequential=state.installstate.sequential, + pypi_mirror=state.pypi_mirror, + ) + if retcode: + ctx.abort() + + +@cli.command(short_help="Uninstalls all packages not specified in Pipfile.lock.") +@option("--dry-run", is_flag=True, default=False, help="Just output unneeded packages.") +@verbose_option +@three_option +@python_option +@pass_state +@pass_context +def clean(ctx, state, dry_run=False, bare=False, user=False): + """Uninstalls all packages not specified in Pipfile.lock.""" + from ..core import do_clean + do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run) + + +# Only invoke the "did you mean" when an argument wasn't passed (it breaks those). +if "-" not in "".join(sys.argv) and len(sys.argv) > 1: + cli = DYMCommandCollection(sources=[cli]) +if __name__ == "__main__": + cli() diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py new file mode 100644 index 0000000000..cf9daa190d --- /dev/null +++ b/pipenv/cli/options.py @@ -0,0 +1,382 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import + +import os + +from click import BOOL as click_booltype +from click import ( + BadParameter, Group, Option, argument, echo, make_pass_decorator, option +) + +from .. import environments +from ..utils import is_valid_url + + +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) + + +class PipenvGroup(Group): + """Custom Group class provides formatted main help""" + + def get_help_option(self, ctx): + from ..core import format_help + + """Override for showing formatted main help via --help and -h options""" + help_options = self.get_help_option_names(ctx) + if not help_options or not self.add_help_option: + return + + def show_help(ctx, param, value): + if value and not ctx.resilient_parsing: + if not ctx.invoked_subcommand: + # legit main help + echo(format_help(ctx.get_help())) + else: + # legit sub-command help + echo(ctx.get_help(), color=ctx.color) + ctx.exit() + + return Option( + help_options, + is_flag=True, + is_eager=True, + expose_value=False, + callback=show_help, + help="Show this message and exit.", + ) + + +class State(object): + def __init__(self): + self.index = None + self.extra_index_urls = [] + self.verbose = False + self.pypi_mirror = None + self.python = None + self.two = None + self.three = None + self.site_packages = False + self.clear = False + self.system = False + self.installstate = InstallState() + + +class InstallState(object): + def __init__(self): + self.dev = False + self.pre = False + self.selective_upgrade = False + self.keep_outdated = False + self.skip_lock = False + self.ignore_pipfile = False + self.sequential = False + self.code = False + self.requirementstxt = None + self.deploy = False + self.packages = [] + self.editables = [] + + +pass_state = make_pass_decorator(State, ensure=True) + + +def index_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.index = value + return value + return option('-i', '--index', expose_value=False, envvar="PIP_INDEX_URL", + help='Target PyPI-compatible package index url.', nargs=1, + callback=callback)(f) + + +def extra_index_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.extra_index_urls.extend(list(value)) + return value + return option("--extra-index-url", multiple=True, expose_value=False, + help=u"URLs to the extra PyPI compatible indexes to query for package lookups.", + callback=callback, envvar="PIP_EXTRA_INDEX_URL")(f) + + +def editable_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.editables.extend(value) + return value + return option('-e', '--editable', expose_value=False, multiple=True, + help='An editable python package URL or path, often to a VCS repo.', + callback=callback)(f) + + +def sequential_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.sequential = value + return value + return option("--sequential", is_flag=True, default=False, expose_value=False, + help="Install dependencies one-at-a-time, instead of concurrently.", + callback=callback, type=click_booltype)(f) + + +def skip_lock_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.skip_lock = value + return value + return option("--skip-lock", is_flag=True, default=False, expose_value=False, + help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", + callback=callback, type=click_booltype)(f) + + +def keep_outdated_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.keep_outdated = value + return value + return option("--keep-outdated", is_flag=True, default=False, expose_value=False, + help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", + callback=callback, type=click_booltype)(f) + + +def selective_upgrade_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.selective_upgrade = value + return value + return option("--selective-upgrade", is_flag=True, default=False, type=click_booltype, + help="Update specified packages.", callback=callback, + expose_value=False)(f) + + +def ignore_pipfile_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.ignore_pipfile = value + return value + return option("--ignore-pipfile", is_flag=True, default=False, expose_value=False, + help="Ignore Pipfile when installing, using the Pipfile.lock.", + callback=callback)(f) + + +def dev_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.dev = value + return value + return option("--dev", "-d", is_flag=True, default=False, type=click_booltype, + help="Install package(s) in [dev-packages].", callback=callback, + expose_value=False)(f) + + +def pre_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.pre = value + return value + return option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.", + callback=callback, type=click_booltype, expose_value=False)(f) + + +def package_arg(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.packages.extend(value) + return value + return argument('packages', nargs=-1, callback=callback, expose_value=False,)(f) + + +def three_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value is not None: + state.three = value + state.two = not value + return value + return option("--three/--two", is_flag=True, default=None, + help="Use Python 3/2 when creating virtualenv.", callback=callback, + expose_value=False)(f) + + +def python_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value is not None: + state.python = validate_python_path(ctx, param, value) + return value + return option("--python", default=False, nargs=1, callback=callback, + help="Specify which version of Python virtualenv should use.", + expose_value=False)(f) + + +def pypi_mirror_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value is not None: + state.pypi_mirror = validate_pypi_mirror(ctx, param, value) + return value + return option("--pypi-mirror", default=environments.PIPENV_PYPI_MIRROR, nargs=1, + callback=callback, help="Specify a PyPI mirror.", expose_value=False)(f) + + +def verbose_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + state.verbose = True + setup_verbosity(ctx, param, value) + return option("--verbose", "-v", is_flag=True, expose_value=False, + callback=callback, help="Verbose mode.")(f) + + +def site_packages_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.site_packages = value + return value + return option("--site-packages", is_flag=True, default=False, type=click_booltype, + help="Enable site-packages for the virtualenv.", callback=callback, + expose_value=False)(f) + + +def clear_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.clear = value + return value + return option("--clear", is_flag=True, callback=callback, type=click_booltype, + help="Clears caches (pipenv, pip, and pip-tools).", + expose_value=False)(f) + + +def system_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value is not None: + state.system = value + return value + return option("--system", is_flag=True, default=False, help="System pip management.", + callback=callback, type=click_booltype, expose_value=False)(f) + + +def requirementstxt_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + state.installstate.requirementstxt = value + return value + return option("--requirements", "-r", nargs=1, default=False, expose_value=False, + help="Import a requirements.txt file.", callback=callback)(f) + + +def requirements_flag(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + state.installstate.requirementstxt = value + return value + return option("--requirements", "-r", default=False, is_flag=True, expose_value=False, + help="Generate output in requirements.txt format.", callback=callback)(f) + + +def code_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + state.installstate.code = value + return value + return option("--code", "-c", nargs=1, default=False, help="Import from codebase.", + callback=callback, expose_value=False)(f) + + +def deploy_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.deploy = value + return value + return option("--deploy", is_flag=True, default=False, type=click_booltype, + help=u"Abort if the Pipfile.lock is out-of-date, or Python version is" + " wrong.", callback=callback, expose_value=False)(f) + + +def setup_verbosity(ctx, param, value): + if not value: + return + import logging + logging.getLogger("pip").setLevel(logging.INFO) + environments.PIPENV_VERBOSITY = 1 + + +def validate_python_path(ctx, param, value): + # Validating the Python path is complicated by accepting a number of + # friendly options: the default will be boolean False to enable + # autodetection but it may also be a value which will be searched in + # the path or an absolute path. To report errors as early as possible + # we'll report absolute paths which do not exist: + if isinstance(value, (str, bytes)): + if os.path.isabs(value) and not os.path.isfile(value): + raise BadParameter("Expected Python at path %s does not exist" % value) + return value + + +def validate_pypi_mirror(ctx, param, value): + if value and not is_valid_url(value): + raise BadParameter("Invalid PyPI mirror URL: %s" % value) + return value + + +def common_options(f): + f = pypi_mirror_option(f) + f = verbose_option(f) + f = clear_option(f) + f = three_option(f) + f = python_option(f) + return f + + +def install_base_options(f): + f = common_options(f) + f = dev_option(f) + f = keep_outdated_option(f) + return f + + +def uninstall_options(f): + f = install_base_options(f) + f = skip_lock_option(f) + f = editable_option(f) + f = package_arg(f) + return f + + +def lock_options(f): + f = install_base_options(f) + f = requirements_flag(f) + f = pre_option(f) + return f + + +def sync_options(f): + f = install_base_options(f) + f = sequential_option(f) + return f + + +def install_options(f): + f = sync_options(f) + f = index_option(f) + f = extra_index_option(f) + f = requirementstxt_option(f) + f = pre_option(f) + f = selective_upgrade_option(f) + f = ignore_pipfile_option(f) + f = editable_option(f) + f = package_arg(f) + return f + + +def general_options(f): + f = common_options(f) + f = site_packages_option(f) + return f diff --git a/pipenv/core.py b/pipenv/core.py index bab12f937d..52c127b10a 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -16,6 +16,7 @@ from blindspin import spinner import six +from .cmdparse import Script from .project import Project, SourceNotFound from .utils import ( convert_deps_to_pip, @@ -36,7 +37,6 @@ is_pinned, is_star, rmtree, - split_argument, fs_str, clean_resolved_dep, ) @@ -655,6 +655,7 @@ def do_install_dependencies( If requirements is True, simply spits out a requirements format to stdout. """ + from .vendor.requirementslib import Requirement def cleanup_procs(procs, concurrent): for c in procs: @@ -672,7 +673,7 @@ def cleanup_procs(procs, concurrent): click.echo( "{0} {1}! Will try again.".format( crayons.red("An error occurred while installing"), - crayons.green(c.dep.split("--hash")[0].strip()), + crayons.green(c.dep.as_line()), ) ) @@ -711,24 +712,18 @@ def cleanup_procs(procs, concurrent): ) failed_deps_list = [] if requirements: - # Comment out packages that shouldn't be included in - # requirements.txt, for pip9. - # Additional package selectors, specific to pip's --hash checking mode. - for l in (deps_list, dev_deps_list): - for i, dep in enumerate(l): - l[i] = list(l[i]) - if "--hash" in l[i][0]: - l[i][0] = l[i][0].split("--hash")[0].strip() index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") + deps_list = [dep for dep, ignore_hash, block in deps_list] + dev_deps_list = [dep for dep, ignore_hash, block in dev_deps_list] # Output only default dependencies click.echo(index_args) if not dev: - click.echo("\n".join(d[0] for d in sorted(deps_list))) + click.echo("\n".join(d.partition('--hash')[0].strip() for d in sorted(deps_list))) sys.exit(0) # Output only dev dependencies if dev: - click.echo("\n".join(d[0] for d in sorted(dev_deps_list))) + click.echo("\n".join(d.partition('--hash')[0].strip() for d in sorted(dev_deps_list))) sys.exit(0) procs = [] deps_list_bar = progress.bar( @@ -737,9 +732,30 @@ def cleanup_procs(procs, concurrent): for dep, ignore_hash, block in deps_list_bar: if len(procs) < PIPENV_MAX_SUBPROCESS: # Use a specific index, if specified. - dep, index = split_argument(dep, short="i", long_="index", num=1) - dep, extra_indexes = split_argument(dep, long_="extra-index-url") + index = None + if ' --index' in dep: + dep, _, index = dep.partition(' --index') + index = index.lstrip('=') + elif ' -i ' in dep: + dep, _, index = dep.partition(' -i ') + extra_indexes = [] + if '--extra-index-url' in dep: + split_dep = dep.split('--extra-index-url') + dep, extra_indexes = split_dep[0], split_dep[1:] + dep = Requirement.from_line(dep) + if index: + _index = None + try: + _index = project.find_source(index).get('name') + except SourceNotFound: + _index = None + dep.index = _index + dep._index = index + dep.extra_indexes = extra_indexes # Install the module. + prev_no_deps_setting = no_deps + if dep.is_file_or_url and any(dep.req.uri.endswith(ext) for ext in ['zip', 'tar.gz']): + no_deps = False c = pip_install( dep, ignore_hashes=ignore_hash, @@ -753,7 +769,10 @@ def cleanup_procs(procs, concurrent): ) c.dep = dep c.ignore_hash = ignore_hash + c.index = index + c.extra_indexes = extra_indexes procs.append(c) + no_deps = prev_no_deps_setting if len(procs) >= PIPENV_MAX_SUBPROCESS or len(procs) == len(deps_list): cleanup_procs(procs, concurrent) procs = [] @@ -765,18 +784,20 @@ def cleanup_procs(procs, concurrent): ) for dep, ignore_hash in progress.bar(failed_deps_list, label=INSTALL_LABEL2): # Use a specific index, if specified. - dep, index = split_argument(dep, short="i", long_="index", num=1) - dep, extra_indexes = split_argument(dep, long_="extra-index-url") # Install the module. + prev_no_deps_setting = no_deps + if dep.is_file_or_url and any(dep.req.uri.endswith(ext) for ext in ['zip', 'tar.gz']): + no_deps = False c = pip_install( dep, ignore_hashes=ignore_hash, allow_global=allow_global, no_deps=no_deps, - index=index, + index=getattr(dep, "_index", None), requirements_dir=requirements_dir, - extra_indexes=extra_indexes, + extra_indexes=getattr(dep, "extra_indexes", None), ) + no_deps = prev_no_deps_setting # The Installation failed… if c.return_code != 0: # We echo both c.out and c.err because pip returns error details on out. @@ -788,7 +809,7 @@ def cleanup_procs(procs, concurrent): click.echo( "{0} {1}{2}".format( crayons.green("Success installing"), - crayons.green(dep.split("--hash")[0].strip()), + crayons.green(dep.name), crayons.green("!"), ) ) @@ -884,7 +905,7 @@ def parse_download_fname(fname, name): if fname.endswith(".tar"): fname, _ = os.path.splitext(fname) # Substring out package name (plus dash) from file name to get version. - version = fname[len(name) + 1 :] + version = fname[len(name) + 1:] # Ignore implicit post releases in version number. if "-" in version and version.split("-")[1].isdigit(): version = version.split("-")[0] @@ -1255,7 +1276,7 @@ def do_init( def pip_install( - package_name=None, + requirement=None, r=None, allow_global=False, ignore_hashes=False, @@ -1269,51 +1290,28 @@ def pip_install( pypi_mirror=None, ): from notpip._internal import logger as piplogger - from notpip._vendor.pyparsing import ParseException - from .vendor.requirementslib import Requirement + src = [] if environments.is_verbose(): - click.echo( - crayons.normal("Installing {0!r}".format(package_name), bold=True), err=True - ) piplogger.setLevel(logging.INFO) + if requirement: + click.echo( + crayons.normal("Installing {0!r}".format(requirement.name), bold=True), + err=True + ) # Create files for hash mode. - if not package_name.startswith("-e ") and (not ignore_hashes) and (r is None): + if requirement and not requirement.editable and (not ignore_hashes) and (r is None): fd, r = tempfile.mkstemp( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir ) with os.fdopen(fd, "w") as f: - f.write(package_name) + f.write(requirement.as_line()) # Install dependencies when a package is a VCS dependency. - try: - req = Requirement.from_line( - package_name.split("--hash")[0].split("--trusted-host")[0] - ).vcs - except (ParseException, ValueError) as e: - click.echo("{0}: {1}".format(crayons.red("WARNING"), e), err=True) - click.echo( - "{0}… You will have to reinstall any packages that failed to install.".format( - crayons.red("ABORTING INSTALL") - ), - err=True, - ) - click.echo( - "You may have to manually run {0} when you are finished.".format( - crayons.normal("pipenv lock", bold=True) - ) - ) - sys.exit(1) - if req: + if requirement and requirement.vcs: no_deps = False # Don't specify a source directory when using --system. if not allow_global and ("PIP_SRC" not in os.environ): - src = "--src {0}".format( - escape_grouped_arguments(project.virtualenv_src_location) - ) - else: - src = "" - else: - src = "" + src.extend(["--src", "{0}".format(project.virtualenv_src_location)]) # Try installing for each source in project.sources. if index: @@ -1341,37 +1339,39 @@ def pip_install( create_mirror_source(pypi_mirror) if is_pypi_url(source["url"]) else source for source in sources ] - if package_name.startswith("-e "): - install_reqs = ' -e "{0}"'.format(package_name.split("-e ")[1]) - elif r: - install_reqs = " -r {0}".format(escape_grouped_arguments(r)) + if (requirement and requirement.editable) or not r: + install_reqs = requirement.as_line(as_list=True) + if requirement.editable and install_reqs[0].startswith("-e "): + req, install_reqs = install_reqs[0], install_reqs[1:] + editable_opt, req = req.split(" ", 1) + install_reqs = [editable_opt, req] + install_reqs + if not any(item.startswith("--hash") for item in install_reqs): + ignore_hashes = True else: - install_reqs = ' "{0}"'.format(package_name) - # Skip hash-checking mode, when appropriate. - if r: + install_reqs = ["-r", r] with open(r) as f: if "--hash" not in f.read(): ignore_hashes = True - else: - if "--hash" not in install_reqs: - ignore_hashes = True + pip_command = [ + which_pip(allow_global=allow_global), + "install" + ] + if pre: + pip_command.append("--pre") + if src: + pip_command.extend(src) + if environments.is_verbose(): + pip_command.append("--verbose") + pip_command.append("--upgrade") + if selective_upgrade: + pip_command.append("--upgrade-strategy=only-if-needed") + if no_deps: + pip_command.append("--no-deps") + pip_command.extend(install_reqs) + pip_command.extend(prepare_pip_source_args(sources)) if not ignore_hashes: - install_reqs += " --require-hashes" - pip_args = { - "no_deps": "--no-deps" if no_deps else "", - "pre": "--pre" if pre else "", - "quoted_pip": escape_grouped_arguments(which_pip(allow_global=allow_global)), - "upgrade_strategy": ( - "--upgrade --upgrade-strategy=only-if-needed" if selective_upgrade else "" - ), - "sources": " ".join(prepare_pip_source_args(sources)), - "src": src, - "verbose_flag": "--verbose" if environments.is_verbose() else "", - "install_reqs": install_reqs - } - pip_command = "{quoted_pip} install {pre} {src} {verbose_flag} {upgrade_strategy} {no_deps} {install_reqs} {sources}".format( - **pip_args - ) + pip_command.append("--require-hashes") + if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) cache_dir = Path(PIPENV_CACHE_DIR) @@ -1380,7 +1380,13 @@ def pip_install( "PIP_WHEEL_DIR": fs_str(cache_dir.joinpath("wheels").as_posix()), "PIP_DESTINATION_DIR": fs_str(cache_dir.joinpath("pkgs").as_posix()), "PIP_EXISTS_ACTION": fs_str("w"), + "PATH": fs_str(os.environ.get("PATH")) } + if src: + pip_config.update({ + "PIP_SRC": fs_str(project.virtualenv_src_location) + }) + pip_command = Script.parse(pip_command).cmdify() c = delegator.run(pip_command, block=block, env=pip_config) return c @@ -1623,8 +1629,10 @@ def do_outdated(pypi_mirror=None): def do_install( - package_name=False, - more_packages=False, + packages=False, + editable_packages=False, + index_url=False, + extra_index_url=False, dev=False, three=False, python=False, @@ -1649,12 +1657,13 @@ def do_install( ) if selective_upgrade: keep_outdated = True - more_packages = more_packages or [] + packages = packages if packages else [] + editable_packages = editable_packages if editable_packages else [] + package_args = [p for p in packages if p] + [p for p in editable_packages if p] + skip_requirements = False # Don't search for requirements.txt files if the user provides one - if requirements or package_name or project.pipfile_exists: + if requirements or package_args or project.pipfile_exists: skip_requirements = True - else: - skip_requirements = False concurrent = not sequential # Ensure that virtualenv is available. ensure_project( @@ -1673,7 +1682,7 @@ def do_install( keep_outdated = project.settings.get("keep_outdated") remote = requirements and is_valid_url(requirements) # Warn and exit if --system is used without a pipfile. - if (system and package_name) and not (PIPENV_VIRTUALENV): + if (system and package_args) and not (PIPENV_VIRTUALENV): click.echo( "{0}: --system is intended to be used for Pipfile installation, " "not installation of specific packages. Aborting.".format( @@ -1757,32 +1766,9 @@ def do_install( for req in import_from_code(code): click.echo(" Found {0}!".format(crayons.green(req))) project.add_package_to_pipfile(req) - # Capture -e argument and assign it to following package_name. - more_packages = list(more_packages) - if package_name == "-e": - if not more_packages: - raise click.BadArgumentUsage("Please provide path to editable package") - package_name = " ".join([package_name, more_packages.pop(0)]) - # capture indexes and extra indexes - line = [package_name] + more_packages - line = " ".join(str(s) for s in line).strip() - index_indicators = ["-i", "--index", "--extra-index-url"] - index, extra_indexes = None, None - if any(line.endswith(s) for s in index_indicators): - # check if cli option is not end of command - raise click.BadArgumentUsage("Please provide index value") - if any(s in line for s in index_indicators): - line, index = split_argument(line, short="i", long_="index", num=1) - line, extra_indexes = split_argument(line, long_="extra-index-url") - package_names = line.split() - package_name = package_names[0] - if len(package_names) > 1: - more_packages = package_names[1:] - else: - more_packages = [] # Capture . argument and assign it to nothing - if package_name == ".": - package_name = False + if len(packages) == 1 and packages[0] == ".": + packages = False # Install editable local packages before locking - this gives us access to dist-info if project.pipfile_exists and ( # double negatives are for english readability, leave them alone. @@ -1793,38 +1779,40 @@ def do_install( project.editable_packages if not dev else project.editable_dev_packages ) for package in section.keys(): - converted = convert_deps_to_pip( + req = convert_deps_to_pip( {package: section[package]}, project=project, r=False ) - if not package_name: - if converted: - package_name = converted.pop(0) - if converted: - more_packages.extend(converted) + if req: + req = req[0] + req = req[len("-e "):] if req.startswith("-e ") else req + if not editable_packages: + editable_packages = [req] + else: + editable_packages.extend([req]) # Allow more than one package to be provided. - package_names = [package_name] + more_packages + package_args = [p for p in packages] + ["-e {0}".format(pkg) for pkg in editable_packages] # Support for --selective-upgrade. # We should do this part first to make sure that we actually do selectively upgrade # the items specified if selective_upgrade: from .vendor.requirementslib import Requirement - for i, package_name in enumerate(package_names[:]): + for i, package in enumerate(package_args[:]): section = project.packages if not dev else project.dev_packages - package = Requirement.from_line(package_name) + package = Requirement.from_line(package) package__name, package__val = package.pipfile_entry try: if not is_star(section[package__name]) and is_star(package__val): # Support for VCS dependencies. - package_names[i] = convert_deps_to_pip( - {package_name: section[package__name]}, project=project, r=False + package_args[i] = convert_deps_to_pip( + {packages: section[package__name]}, project=project, r=False )[0] except KeyError: pass # Install all dependencies, if none was provided. # This basically ensures that we have a pipfile and lockfile, then it locks and # installs from the lockfile - if package_name is False: + if packages is False and editable_packages is False: # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) @@ -1844,36 +1832,40 @@ def do_install( # This is for if the user passed in dependencies, then we want to maek sure we else: from .vendor.requirementslib import Requirement + # make a tuple of (display_name, entry) + pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] - for package_name in package_names: + for pkg_line in pkg_list: click.echo( crayons.normal( - u"Installing {0}…".format(crayons.green(package_name, bold=True)), + u"Installing {0}…".format(crayons.green(pkg_line, bold=True)), bold=True, ) ) # pip install: with spinner(): + try: + pkg_requirement = Requirement.from_line(pkg_line) + except ValueError as e: + click.echo("{0}: {1}".format(crayons.red("WARNING"), e)) + requirements_directory.cleanup() + sys.exit(1) + if index_url: + pkg_requirement.index = index_url c = pip_install( - package_name, + pkg_requirement, ignore_hashes=True, allow_global=system, selective_upgrade=selective_upgrade, no_deps=False, pre=pre, requirements_dir=requirements_directory.name, - index=index, - extra_indexes=extra_indexes, + index=index_url, + extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) # Warn if --editable wasn't passed. - try: - converted = Requirement.from_line(package_name) - except ValueError as e: - click.echo("{0}: {1}".format(crayons.red("WARNING"), e)) - requirements_directory.cleanup() - sys.exit(1) - if converted.is_vcs and not converted.editable: + if pkg_requirement.is_vcs and not pkg_requirement.editable: click.echo( "{0}: You installed a VCS dependency in non-editable mode. " "This will work fine, but sub-dependencies will not be resolved by {1}." @@ -1890,7 +1882,7 @@ def do_install( except AssertionError: click.echo( "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(package_name) + crayons.red("Error: ", bold=True), crayons.green(pkg_line) ), err=True, ) @@ -1899,7 +1891,7 @@ def do_install( click.echo( "This is likely caused by a bug in {0}. " "Report this to its maintainers.".format( - crayons.green(package_name) + crayons.green(pkg_requirement.name) ), err=True, ) @@ -1908,7 +1900,7 @@ def do_install( click.echo( "{0} {1} {2} {3}{4}".format( crayons.normal("Adding", bold=True), - crayons.green(package_name, bold=True), + crayons.green(pkg_requirement.name, bold=True), crayons.normal("to Pipfile's", bold=True), crayons.red("[dev-packages]" if dev else "[packages]", bold=True), crayons.normal("…", bold=True), @@ -1916,7 +1908,7 @@ def do_install( ) # Add the package to the Pipfile. try: - project.add_package_to_pipfile(package_name, dev) + project.add_package_to_pipfile(pkg_requirement, dev) except ValueError as e: click.echo( "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), e) @@ -1940,8 +1932,8 @@ def do_install( def do_uninstall( - package_name=False, - more_packages=False, + packages=False, + editable_packages=False, three=None, python=False, system=False, @@ -1952,13 +1944,21 @@ def do_uninstall( pypi_mirror=None, ): from .environments import PIPENV_USE_SYSTEM + from .vendor.requirementslib import Requirement # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True # Ensure that virtualenv is available. + # TODO: We probably shouldn't ensure a project exists if the outcome will be to just + # install things in order to remove them... maybe tell the user to install first? ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) - package_names = (package_name,) + more_packages + editable_pkgs = [ + Requirement.from_line("-e {0}".format(p)).name + for p in editable_packages + if p + ] + package_names = [p for p in packages if p] + editable_pkgs pipfile_remove = True # Un-install all dependencies, if --all was provided. if all is True: @@ -1966,7 +1966,7 @@ def do_uninstall( crayons.normal(u"Un-installing all packages from virtualenv…", bold=True) ) do_purge(allow_global=system) - sys.exit(0) + return # Uninstall [dev-packages], if --dev was provided. if all_dev: if "dev-packages" not in project.parsed_pipfile: @@ -1976,16 +1976,16 @@ def do_uninstall( bold=True, ) ) - sys.exit(0) + return click.echo( crayons.normal( u"Un-installing {0}…".format(crayons.red("[dev-packages]")), bold=True ) ) package_names = project.dev_packages.keys() - if package_name is False and not all_dev: + if packages is False and editable_packages is False and not all_dev: click.echo(crayons.red("No package provided!"), err=True) - sys.exit(1) + return 1 for package_name in package_names: click.echo(u"Un-installing {0}…".format(crayons.green(package_name))) cmd = "{0} uninstall {1} -y".format( @@ -2444,7 +2444,7 @@ def do_sync( ), err=True, ) - sys.exit(1) + return 1 # Ensure that virtualenv is available if not system. ensure_project( diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index e3156f6a6b..9e74156055 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -283,21 +283,20 @@ def get_legacy_dependencies(self, ireq): from pipenv.utils import chdir with chdir(ireq.setup_py_dir): from setuptools.dist import distutils - distutils.core.run_setup(ireq.setup_py) + dist = distutils.core.run_setup(ireq.setup_py) except (ImportError, InstallationError, TypeError, AttributeError): pass try: - dist = ireq.get_dist() + dist = ireq.get_dist() if not dist else dist except InstallationError: ireq.run_egg_info() dist = ireq.get_dist() except (TypeError, ValueError, AttributeError): pass else: - if dist.has_metadata('requires.txt'): - setup_requires = self.finder.get_extras_links( - dist.get_metadata_lines('requires.txt') - ) + setup_requires = getattr(dist, "extras_require", None) + if not setup_requires: + setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} try: # Pip 9 and below reqset = RequirementSet( diff --git a/pipenv/project.py b/pipenv/project.py index c15dc4cf31..f9decbf641 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -289,7 +289,7 @@ def get_installed_packages(self): new_path = [new_path[0], PIPENV_ROOT, PIPENV_PATCHED, PIPENV_VENDOR] + new_path[1:] sys.path = new_path os.environ['VIRTUAL_ENV'] = self.virtualenv_location - from .patched.notpip._internal.utils.misc import get_installed_distributions + from .vendor.pip_shims.shims import get_installed_distributions return get_installed_distributions(local_only=True) else: return [] @@ -372,7 +372,10 @@ def virtualenv_location(self): @property def virtualenv_src_location(self): - loc = os.sep.join([self.virtualenv_location, "src"]) + if self.virtualenv_location: + loc = os.sep.join([self.virtualenv_location, "src"]) + else: + loc = os.sep.join([self.project_directory, "src"]) mkdir_p(loc) return loc @@ -767,13 +770,14 @@ def remove_package_from_pipfile(self, package_name, dev=False): del p[key][name] self.write_toml(p) - def add_package_to_pipfile(self, package_name, dev=False): + def add_package_to_pipfile(self, package, dev=False): from .vendor.requirementslib import Requirement # Read and append Pipfile. p = self.parsed_pipfile # Don't re-capitalize file URLs or VCSs. - package = Requirement.from_line(package_name.strip()) + if not isinstance(package, Requirement): + package = Requirement.from_line(package.strip()) _, converted = package.pipfile_entry key = "dev-packages" if dev else "packages" # Set empty group if it doesn't exist yet. diff --git a/pipenv/utils.py b/pipenv/utils.py index 12750fdac4..b7295c2921 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1072,38 +1072,6 @@ def handle_remove_readonly(func, path, exc): raise -def split_argument(req, short=None, long_=None, num=-1): - """Split an argument from a string (finds None if not present). - - Uses -short , --long , and --long=arg as permutations. - - returns string, index - """ - index_entries = [] - import re - - if long_: - index_entries.append("--{0}".format(long_)) - if short: - index_entries.append("-{0}".format(short)) - match_string = "|".join(index_entries) - matches = re.findall("(?<=\s)({0})([\s=])(\S+)".format(match_string), req) - remove_strings = [] - match_values = [] - for match in matches: - match_values.append(match[-1]) - remove_strings.append("".join(match)) - for string_to_remove in remove_strings: - req = req.replace(" {0}".format(string_to_remove), "") - if not match_values: - return req, None - if num == 1: - return req, match_values[0] - if num == -1: - return req, match_values - return req, match_values[:num] - - @contextmanager def atomic_open_for_write(target, binary=False, newline=None, encoding=None): """Atomically open `target` for writing. @@ -1167,22 +1135,6 @@ def extract_uri_from_vcs_dep(dep): return None -def resolve_ref(vcs_obj, target_dir, ref): - return vcs_obj.get_revision_sha(target_dir, ref) - - -def obtain_vcs_req(vcs_obj, src_dir, name, rev=None): - target_dir = os.path.join(src_dir, name) - target_rev = vcs_obj.make_rev_options(rev) - if not os.path.exists(target_dir): - vcs_obj.obtain(target_dir) - if not vcs_obj.is_commit_id_equal( - target_dir, rev - ) and not vcs_obj.is_commit_id_equal(target_dir, target_rev): - vcs_obj.update(target_dir, target_rev) - return vcs_obj.get_revision(target_dir) - - def get_vcs_deps( project, pip_freeze=None, @@ -1193,8 +1145,8 @@ def get_vcs_deps( dev=False, pypi_mirror=None, ): - from .patched.notpip._internal.vcs import VcsSupport from ._compat import TemporaryDirectory, Path + import atexit from .vendor.requirementslib import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" @@ -1204,27 +1156,23 @@ def get_vcs_deps( packages = getattr(project, section) except AttributeError: return [], [] - if not os.environ.get("PIP_SRC") and not project.virtualenv_location: - _src_dir = TemporaryDirectory(prefix="pipenv-", suffix="-src") - src_dir = Path(_src_dir.name) - else: + if os.environ.get("PIP_SRC"): src_dir = Path( os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) ) src_dir.mkdir(mode=0o775, exist_ok=True) - vcs_registry = VcsSupport + else: + src_dir = TemporaryDirectory(prefix="pipenv-lock-dir") + atexit.register(src_dir.cleanup) for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) - backend = vcs_registry()._registry.get(requirement.vcs) - __vcs = backend(url=requirement.req.vcs_uri) - locked_rev = None name = requirement.normalized_name - locked_rev = obtain_vcs_req( - __vcs, src_dir.as_posix(), name, rev=pkg_pipfile.get("ref") - ) + commit_hash = None if requirement.is_vcs: - requirement.req.ref = locked_rev - lockfile[name] = requirement.pipfile_entry[1] + with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo: + commit_hash = repo.get_commit_hash() + lockfile[name] = requirement.pipfile_entry[1] + lockfile[name]['ref'] = commit_hash reqs.append(requirement) return reqs, lockfile @@ -1243,17 +1191,24 @@ def translate_markers(pipfile_entry): if not isinstance(pipfile_entry, Mapping): raise TypeError("Entry is not a pipfile formatted mapping.") from notpip._vendor.distlib.markers import DEFAULT_CONTEXT as marker_context + from .vendor.packaging.markers import Marker + from .vendor.vistir.misc import dedup allowed_marker_keys = ["markers"] + [k for k in marker_context.keys()] provided_keys = list(pipfile_entry.keys()) if hasattr(pipfile_entry, "keys") else [] - pipfile_marker = next((k for k in provided_keys if k in allowed_marker_keys), None) + pipfile_markers = [k for k in provided_keys if k in allowed_marker_keys] new_pipfile = dict(pipfile_entry).copy() - if pipfile_marker: - entry = "{0}".format(pipfile_entry[pipfile_marker]) - if pipfile_marker != "markers": - entry = "{0} {1}".format(pipfile_marker, entry) - new_pipfile.pop(pipfile_marker) - new_pipfile["markers"] = entry + marker_set = set() + if "markers" in new_pipfile: + marker_set.add(str(Marker(new_pipfile.get("markers")))) + for m in pipfile_markers: + entry = "{0}".format(pipfile_entry[m]) + if m != "markers": + marker_set.add(str(Marker("{0}{1}".format(m, entry)))) + new_pipfile.pop(m) + if marker_set: + new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) if " and " in s else s + for s in sorted(dedup(marker_set))]))).replace('"', "'") return new_pipfile diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 910db3d516..0faea40b4e 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.1.5' +__version__ = '1.1.7.dev0' from .exceptions import RequirementError diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 9b6d32e95c..16fc4ba8af 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -2,25 +2,20 @@ from __future__ import absolute_import, print_function, unicode_literals import copy -import errno import hashlib import json import os import six import sys -from contextlib import contextmanager - import requests +import pip_shims import vistir from appdirs import user_cache_dir from packaging.requirements import Requirement -from pip_shims.shims import ( - FAVORITE_HASH, Link, SafeFileCache, VcsSupport, is_file_url, url_to_path -) -from .utils import as_tuple, key_from_req, lookup_table +from .utils import as_tuple, key_from_req, lookup_table, get_pinned_version if six.PY2: @@ -194,22 +189,24 @@ def _reverse_dependencies(self, cache_keys): for dep_name in self.cache[name][version_and_extras]) -class HashCache(SafeFileCache): - """Caches hashes of PyPI artifacts so we do not need to re-download them +class HashCache(pip_shims.SafeFileCache): + """Caches hashes of PyPI artifacts so we do not need to re-download them. - Hashes are only cached when the URL appears to contain a hash in it and the cache key includes - the hash value returned from the server). This ought to avoid ssues where the location on the - server changes.""" + Hashes are only cached when the URL appears to contain a hash in it and the + cache key includes the hash value returned from the server). This ought to + avoid ssues where the location on the server changes. + """ def __init__(self, *args, **kwargs): session = kwargs.pop('session', requests.session()) + cache_dir = kwargs.pop('cache_dir', CACHE_DIR) self.session = session - kwargs.setdefault('directory', os.path.join(CACHE_DIR, 'hash-cache')) + kwargs.setdefault('directory', os.path.join(cache_dir, 'hash-cache')) super(HashCache, self).__init__(*args, **kwargs) def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs = VcsSupport() + vcs = pip_shims.VcsSupport() orig_scheme = location.scheme new_location = copy.deepcopy(location) if orig_scheme in vcs.all_schemes: @@ -217,7 +214,7 @@ def get_hash(self, location): can_hash = new_location.hash if can_hash: # hash url WITH fragment - hash_value = self.get(new_location.url) + hash_value = self._get_file_hash(new_location.url) if not new_location.url.startswith("ssh") else None if not hash_value: hash_value = self._get_file_hash(new_location) hash_value = hash_value.encode('utf8') @@ -226,41 +223,117 @@ def get_hash(self, location): return hash_value.decode('utf8') def _get_file_hash(self, location): - h = hashlib.new(FAVORITE_HASH) - with open_local_or_remote_file(location, self.session) as fp: + h = hashlib.new(pip_shims.FAVORITE_HASH) + with vistir.contextmanagers.open_file(location, self.session) as fp: for chunk in iter(lambda: fp.read(8096), b""): h.update(chunk) - return ":".join([FAVORITE_HASH, h.hexdigest()]) + return ":".join([pip_shims.FAVORITE_HASH, h.hexdigest()]) -@contextmanager -def open_local_or_remote_file(link, session): - """ - Open local or remote file for reading. +class _JSONCache(object): + """A persistent cache backed by a JSON file. + + The cache file is written to the appropriate user cache dir for the + current platform, i.e. + + ~/.cache/pip-tools/depcache-pyX.Y.json - :type link: pip._internal.index.Link - :type session: requests.Session - :raises ValueError: If link points to a local directory. - :return: a context manager to the opened file-like object + Where X.Y indicates the Python version. """ - if isinstance(link, Link): - url = link.url_without_fragment - else: - url = link - - if is_file_url(link): - # Local URL - local_path = url_to_path(url) - if os.path.isdir(local_path): - raise ValueError("Cannot open directory for read: {}".format(url)) + filename_format = None + + def __init__(self, cache_dir=CACHE_DIR): + vistir.mkdir_p(cache_dir) + python_version = ".".join(str(digit) for digit in sys.version_info[:2]) + cache_filename = self.filename_format.format( + python_version=python_version, + ) + self._cache_file = os.path.join(cache_dir, cache_filename) + self._cache = None + + @property + def cache(self): + """The dictionary that is the actual in-memory cache. + + This property lazily loads the cache from disk. + """ + if self._cache is None: + self.read_cache() + return self._cache + + def as_cache_key(self, ireq): + """Given a requirement, return its cache key. + + This behavior is a little weird in order to allow backwards + compatibility with cache files. For a requirement without extras, this + will return, for example:: + + ("ipython", "2.1.0") + + For a requirement with extras, the extras will be comma-separated and + appended to the version, inside brackets, like so:: + + ("ipython", "2.1.0[nbconvert,notebook]") + """ + extras = tuple(sorted(ireq.extras)) + if not extras: + extras_string = "" else: - with open(local_path, 'rb') as local_file: - yield local_file - else: - # Remote URL - headers = {"Accept-Encoding": "identity"} - response = session.get(url, headers=headers, stream=True) + extras_string = "[{}]".format(",".join(extras)) + name = key_from_req(ireq.req) + version = get_pinned_version(ireq) + return name, "{}{}".format(version, extras_string) + + def read_cache(self): + """Reads the cached contents into memory. + """ + if os.path.exists(self._cache_file): + self._cache = read_cache_file(self._cache_file) + else: + self._cache = {} + + def write_cache(self): + """Writes the cache to disk as JSON. + """ + doc = { + '__format__': 1, + 'dependencies': self._cache, + } + with open(self._cache_file, 'w') as f: + json.dump(doc, f, sort_keys=True) + + def clear(self): + self._cache = {} + self.write_cache() + + def __contains__(self, ireq): + pkgname, pkgversion_and_extras = self.as_cache_key(ireq) + return pkgversion_and_extras in self.cache.get(pkgname, {}) + + def __getitem__(self, ireq): + pkgname, pkgversion_and_extras = self.as_cache_key(ireq) + return self.cache[pkgname][pkgversion_and_extras] + + def __setitem__(self, ireq, values): + pkgname, pkgversion_and_extras = self.as_cache_key(ireq) + self.cache.setdefault(pkgname, {}) + self.cache[pkgname][pkgversion_and_extras] = values + self.write_cache() + + def __delitem__(self, ireq): + pkgname, pkgversion_and_extras = self.as_cache_key(ireq) try: - yield response.raw - finally: - response.close() + del self.cache[pkgname][pkgversion_and_extras] + except KeyError: + return + self.write_cache() + + def get(self, ireq, default=None): + pkgname, pkgversion_and_extras = self.as_cache_key(ireq) + return self.cache.get(pkgname, {}).get(pkgversion_and_extras, default) + + +class RequiresPythonCache(_JSONCache): + """Cache a candidate's Requires-Python information. + """ + filename_format = "pyreqcache-py{python_version}.json" diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/vendor/requirementslib/models/markers.py index 534978ed40..83b44b6344 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/vendor/requirementslib/models/markers.py @@ -82,12 +82,12 @@ def from_pipfile(cls, name, pipfile): marker_strings = ["{0} {1}".format(k, pipfile[k]) for k in found_keys] if pipfile.get("markers"): marker_strings.append(pipfile.get("markers")) - markers = [] + markers = set() for marker in marker_strings: - markers.append(marker) + markers.add(marker) combined_marker = None try: - combined_marker = cls.make_marker(" and ".join(markers)) + combined_marker = cls.make_marker(" and ".join(sorted(markers))) except RequirementError: pass else: diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 2bfd8996bc..3a6f5b1ee8 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -import os - from vistir.compat import Path from .requirements import Requirement @@ -30,10 +28,10 @@ def load(cls, path): with pipfile_path.open(encoding="utf-8") as fp: pipfile = super(Pipfile, cls).load(fp) pipfile.dev_requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.dev_packages.items() + Requirement.from_pipfile(k, v) for k, v in pipfile.get("dev-packages", {}).items() ] pipfile.requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.packages.items() + Requirement.from_pipfile(k, v) for k, v in pipfile.get("packages", {}).items() ] pipfile.path = pipfile_path return pipfile @@ -57,10 +55,10 @@ def load(cls, path): def dev_packages(self, as_requirements=True): if as_requirements: return self.dev_requirements - return self.dev_packages + return self.get('dev-packages', {}) @property def packages(self, as_requirements=True): if as_requirements: return self.requirements - return self.packages + return self.get('packages', {}) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 03c351847a..13a3a60ec9 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1,30 +1,35 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import atexit import collections import hashlib import os +from contextlib import contextmanager + import attr -import atexit from first import first from packaging.markers import Marker +from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from six.moves.urllib import parse as urllib_parse -from six.moves.urllib.parse import unquote - from pip_shims.shims import ( InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url, url_to_path ) +from six.moves.urllib import parse as urllib_parse +from six.moves.urllib.parse import unquote from vistir.compat import FileNotFoundError, Path, TemporaryDirectory from vistir.misc import dedup -from vistir.path import get_converted_relative_path, is_valid_url, is_file_url, mkdir_p +from vistir.path import ( + create_tracked_tempdir, get_converted_relative_path, is_file_url, + is_valid_url, mkdir_p +) from ..exceptions import RequirementError -from ..utils import VCS_LIST, is_vcs, is_installable_file +from ..utils import VCS_LIST, is_installable_file, is_vcs from .baserequirement import BaseRequirement from .dependencies import ( AbstractDependency, find_all_matches, get_abstract_dependencies, @@ -32,15 +37,14 @@ ) from .markers import PipenvMarkers from .utils import ( - HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, filter_none, - format_requirement, get_version, init_requirement, + HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string, + filter_none, format_requirement, get_version, init_requirement, is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, specs_to_string, split_markers_from_line, split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs, extras_to_string + validate_specifiers, validate_vcs ) from .vcs import VCSRepository -from packaging.requirements import Requirement as PackagingRequirement @attr.s @@ -475,11 +479,15 @@ class VCSRequirement(FileRequirement): # : vcs reference name (branch / commit / tag) ref = attr.ib(default=None) subdirectory = attr.ib(default=None) + _repo = attr.ib(default=None) name = attr.ib() link = attr.ib() req = attr.ib() def __attrs_post_init__(self): + if not self.uri: + if self.path: + self.uri = path_to_url(self.path) split = urllib_parse.urlsplit(self.uri) scheme, rest = split[0], split[1:] vcs_type = "" @@ -492,9 +500,10 @@ def __attrs_post_init__(self): @link.default def get_link(self): + uri = self.uri if self.uri else path_to_url(self.path) return build_vcs_link( self.vcs, - add_ssh_scheme_to_git_uri(self.uri), + add_ssh_scheme_to_git_uri(uri), name=self.name, ref=self.ref, subdirectory=self.subdirectory, @@ -516,44 +525,6 @@ def vcs_uri(self): uri = "{0}+{1}".format(self.vcs, uri) return uri - def get_commit_hash(self, src_dir=None): - src_dir = os.environ.get('SRC_DIR', None) if not src_dir else src_dir - if not src_dir: - _src_dir = TemporaryDirectory() - atexit.register(_src_dir.cleanup) - src_dir = _src_dir.name - checkout_dir = Path(src_dir).joinpath(self.name).as_posix() - vcsrepo = VCSRepository( - url=self.link.url, - name=self.name, - ref=self.ref if self.ref else None, - checkout_directory=checkout_dir, - vcs_type=self.vcs - ) - vcsrepo.obtain() - return vcsrepo.get_commit_hash() - - def update_repo(self, src_dir=None, ref=None): - src_dir = os.environ.get('SRC_DIR', None) if not src_dir else src_dir - if not src_dir: - _src_dir = TemporaryDirectory() - atexit.register(_src_dir.cleanup) - src_dir = _src_dir.name - checkout_dir = Path(src_dir).joinpath(self.name).as_posix() - ref = self.ref if not ref else ref - vcsrepo = VCSRepository( - url=self.link.url, - name=self.name, - ref=ref if ref else None, - checkout_directory=checkout_dir, - vcs_type=self.vcs - ) - if not os.path.exists(checkout_dir): - vcsrepo.obtain() - else: - vcsrepo.update() - return vcsrepo.get_commit_hash() - @req.default def get_requirement(self): name = self.name or self.link.egg_fragment @@ -586,6 +557,71 @@ def get_requirement(self): req.url = self.uri return req + @property + def is_local(self): + if is_file_url(self.uri): + return True + return False + + @property + def repo(self): + if self._repo is None: + self._repo = self.get_vcs_repo() + return self._repo + + def get_checkout_dir(self, src_dir=None): + src_dir = os.environ.get('PIP_SRC', None) if not src_dir else src_dir + checkout_dir = None + if self.is_local: + path = self.path + if not path: + path = url_to_path(self.uri) + if path and os.path.exists(path): + checkout_dir = os.path.abspath(path) + return checkout_dir + return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) + + def get_vcs_repo(self, src_dir=None): + checkout_dir = self.get_checkout_dir(src_dir=src_dir) + url = "{0}#egg={1}".format(self.vcs_uri, self.name) + vcsrepo = VCSRepository( + url=url, + name=self.name, + ref=self.ref if self.ref else None, + checkout_directory=checkout_dir, + vcs_type=self.vcs + ) + if not self.is_local: + vcsrepo.obtain() + return vcsrepo + + def get_commit_hash(self): + hash_ = None + hash_ = self.repo.get_commit_hash() + return hash_ + + def update_repo(self, src_dir=None, ref=None): + if ref: + self.ref = ref + else: + if self.ref: + ref = self.ref + repo_hash = None + if not self.is_local and ref is not None: + self.repo.checkout_ref(ref) + repo_hash = self.repo.get_commit_hash() + return repo_hash + + @contextmanager + def locked_vcs_repo(self, src_dir=None): + vcsrepo = self.get_vcs_repo(src_dir=src_dir) + if self.ref and not self.is_local: + vcsrepo.checkout_ref(self.ref) + self.ref = self.get_commit_hash() + self.req.revision = self.ref + yield vcsrepo + self._repo = vcsrepo + @classmethod def from_pipfile(cls, name, pipfile): creation_args = {} @@ -602,13 +638,15 @@ def from_pipfile(cls, name, pipfile): pipfile[key] = sorted(dedup([extra.lower() for extra in extras])) if key in VCS_LIST: creation_args["vcs"] = key - composed_uri = add_ssh_scheme_to_git_uri( - "{0}+{1}".format(key, pipfile.get(key)) - ).split("+", 1)[1] - url_keys = [pipfile.get(key), composed_uri] - is_url = any(validity_fn(url_key) for url_key in url_keys for validity_fn in [is_valid_url, is_file_url]) - target_key = "uri" if is_url else "path" - creation_args[target_key] = pipfile.get(key) + target = pipfile.get(key) + drive, path = os.path.splitdrive(target) + if not drive and not os.path.exists(target) and (is_valid_url(target) or + is_file_url(target) or target.startswith('git@')): + creation_args["uri"] = target + else: + creation_args["path"] = target + if os.path.isabs(target): + creation_args["uri"] = path_to_url(target) else: creation_args[key] = pipfile.get(key) creation_args["name"] = name @@ -648,7 +686,13 @@ def from_line(cls, line, editable=None, extras=None): @property def line_part(self): """requirements.txt compatible line part sans-extras""" - if self.req: + if self.is_local: + base_link = self.link + if not self.link: + base_link = self.get_link() + final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) if base_link.egg_fragment else "{0}" + base = final_format.format(self.vcs_uri) + elif self.req: base = self.req.line if base and self.extras and not extras_to_string(self.extras) in base: if self.subdirectory: @@ -673,7 +717,7 @@ def _choose_vcs_source(pipfile): @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=filter_none).copy() + pipfile_dict = attr.asdict(self, filter=lambda k, v: bool(v) is True and k.name != '_repo').copy() if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) name, _ = _strip_extras(pipfile_dict.pop("name")) @@ -702,12 +746,16 @@ def get_name(self): def requirement(self): return self.req.req - @property - def hashes_as_pip(self): + def get_hashes_as_pip(self, as_list=False): if self.hashes: + if as_list: + return [HASH_STRING.format(h) for h in self.hashes] return "".join([HASH_STRING.format(h) for h in self.hashes]) + return "" if not as_list else [] - return "" + @property + def hashes_as_pip(self): + self.get_hashes_as_pip() @property def markers_as_pip(self): @@ -727,7 +775,10 @@ def extras_as_pip(self): def commit_hash(self): if not self.is_vcs: return None - return self.req.get_commit_hash() + commit_hash = None + with self.req.locked_vcs_repo() as repo: + commit_hash = repo.get_commit_hash() + return commit_hash @specifiers.default def get_specifiers(self): @@ -881,15 +932,16 @@ def from_pipfile(cls, name, pipfile): cls_inst.req.req.line = cls_inst.as_line() return cls_inst - def as_line(self, sources=None, include_hashes=True, include_extras=True): + def as_line(self, sources=None, include_hashes=True, include_extras=True, as_list=False): """Format this requirement as a line in requirements.txt. - If `sources` provided, it should be an sequence of mappings, containing + If ``sources`` provided, it should be an sequence of mappings, containing all possible sources to be used for this requirement. - If `sources` is omitted or falsy, no index information will be included + If ``sources`` is omitted or falsy, no index information will be included in the requirement line. """ + include_specifiers = True if self.specifiers else False if self.is_vcs: include_extras = False @@ -901,15 +953,28 @@ def as_line(self, sources=None, include_hashes=True, include_extras=True): self.specifiers if include_specifiers else "", self.markers_as_pip, ] + if as_list: + # This is used for passing to a subprocess call + parts = ["".join(parts)] if include_hashes: - parts.append(self.hashes_as_pip) + hashes = self.get_hashes_as_pip(as_list=as_list) + if as_list: + parts.extend(hashes) + else: + parts.append(hashes) if sources and not (self.requirement.local_file or self.vcs): from ..utils import prepare_pip_source_args if self.index: sources = [s for s in sources if s.get("name") == self.index] - index_string = " ".join(prepare_pip_source_args(sources)) - parts.extend([" ", index_string]) + source_list = prepare_pip_source_args(sources) + if as_list: + parts.extend(sources) + else: + index_string = " ".join(source_list) + parts.extend([" ", index_string]) + if as_list: + return parts line = "".join(parts) return line diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 6fd55b6ff2..6320236ee9 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -117,7 +117,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pipenv.patched.notpip format""" + """Cleans VCS uris from pip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: @@ -154,7 +154,7 @@ def validate_vcs(instance, attr_, value): def validate_path(instance, attr_, value): if not os.path.exists(value): - raise ValueError("Invalid path {0!r}", format(value)) + raise ValueError("Invalid path {0!r}".format(value)) def validate_markers(instance, attr_, value): @@ -256,9 +256,8 @@ def format_specifier(ireq): return ','.join(str(s) for s in specs) or '' -def is_pinned_requirement(ireq): - """ - Returns whether an InstallRequirement is a "pinned" requirement. +def get_pinned_version(ireq): + """Get the pinned version of an InstallRequirement. An InstallRequirement is considered pinned if: @@ -272,18 +271,54 @@ def is_pinned_requirement(ireq): django>1.8 # NOT pinned django~=1.8 # NOT pinned django==1.* # NOT pinned + + Raises `TypeError` if the input is not a valid InstallRequirement, or + `ValueError` if the InstallRequirement is not pinned. """ - if ireq.editable: - return False + try: + specifier = ireq.specifier + except AttributeError: + raise TypeError("Expected InstallRequirement, not {}".format( + type(ireq).__name__, + )) - specifier = getattr(ireq, "specifier", None) + if ireq.editable: + raise ValueError("InstallRequirement is editable") if not specifier: - return False + raise ValueError("InstallRequirement has no version specification") if len(specifier._specs) != 1: - return False + raise ValueError("InstallRequirement has multiple specifications") + + op, version = next(iter(specifier._specs))._spec + if op not in ('==', '===') or version.endswith('.*'): + raise ValueError("InstallRequirement not pinned (is {0!r})".format( + op + version, + )) + + return version + - op, version = first(specifier._specs)._spec - return (op == '==' or op == '===') and not version.endswith('.*') +def is_pinned_requirement(ireq): + """Returns whether an InstallRequirement is a "pinned" requirement. + + An InstallRequirement is considered pinned if: + + - Is not editable + - It has exactly one specifier + - That specifier is "==" + - The version does not contain a wildcard + + Examples: + django==1.8 # pinned + django>1.8 # NOT pinned + django~=1.8 # NOT pinned + django==1.* # NOT pinned + """ + try: + get_pinned_version(ireq) + except (TypeError, ValueError): + return False + return True def as_tuple(ireq): @@ -473,3 +508,5 @@ def fix_requires_python_marker(requires_python): ]) marker_to_add = PackagingRequirement('fakepkg; {0}'.format(marker_str)).marker return marker_to_add + + diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index a588f62985..5d3ec08fc9 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- import attr -from pip_shims import VcsSupport +from pip_shims import VcsSupport, parse_version, pip_version +import vistir import os @@ -22,6 +23,13 @@ def get_repo_instance(self): backend = VCS_SUPPORT._registry.get(self.vcs_type) return backend(url=self.url) + @property + def is_local(self): + url = self.url + if '+' in url: + url = url.split('+')[1] + return url.startswith("file") + def obtain(self): if not os.path.exists(self.checkout_directory): self.repo_instance.obtain(self.checkout_directory) @@ -29,20 +37,29 @@ def obtain(self): self.checkout_ref(self.ref) self.commit_sha = self.get_commit_hash(self.ref) else: - self.ref = self.repo_instance.default_arg_rev if not self.commit_sha: self.commit_sha = self.get_commit_hash() def checkout_ref(self, ref): - target_rev = self.repo_instance.make_rev_options(ref) if not self.repo_instance.is_commit_id_equal( self.checkout_directory, self.get_commit_hash(ref) ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref): - self.repo_instance.switch(self.checkout_directory, self.url, target_rev) + if not self.is_local: + self.update(ref) def update(self, ref): - target_rev = self.repo_instance.make_rev_options(ref) - self.repo_instance.update(self.checkout_directory, target_rev) + target_ref = self.repo_instance.make_rev_options(ref) + sha = self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) + target_rev = target_ref.make_new(sha) + if parse_version(pip_version) > parse_version("18.0"): + self.repo_instance.update(self.checkout_directory, self.url, target_rev) + else: + self.repo_instance.update(self.checkout_directory, target_ref) + self.commit_hash = self.get_commit_hash(ref) def get_commit_hash(self, ref=None): + if ref: + target_ref = self.repo_instance.make_rev_options(ref) + return self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) + # return self.repo_instance.get_revision(self.checkout_directory) return self.repo_instance.get_revision(self.checkout_directory) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 9b6d7bea8a..d1667eeb00 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -27,7 +27,7 @@ requests==2.19.1 idna==2.7 urllib3==1.23 certifi==2018.8.24 -requirementslib==1.1.5 +requirementslib==1.1.6 attrs==18.1.0 distlib==0.2.7 packaging==17.1 diff --git a/setup.cfg b/setup.cfg index f0cb3bf581..78140fa0c0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,6 @@ lines_between_types=1 multi_line_output=5 not_skip=__init__.py known_first_party = - passa + pipenv tests ignore_trailing_comma=true diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 7e8a14422d..6dff468ac8 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -4,22 +4,22 @@ index 4e6174c..75f9b49 100644 +++ b/pipenv/patched/piptools/locations.py @@ -2,10 +2,13 @@ import os from shutil import rmtree - + from .click import secho -from ._compat import user_cache_dir +# Patch by vphilippon 2017-11-22: Use pipenv cache path. +# from ._compat import user_cache_dir +from pipenv.environments import PIPENV_CACHE_DIR - + # The user_cache_dir helper comes straight from pip itself -CACHE_DIR = user_cache_dir('pip-tools') +# CACHE_DIR = user_cache_dir(os.path.join('pip-tools')) +CACHE_DIR = PIPENV_CACHE_DIR - + # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..91902dc 100644 +index 1c4b943..84077f0 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,9 +1,10 @@ @@ -230,7 +230,7 @@ index 1c4b943..91902dc 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +273,47 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +273,46 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -245,21 +245,20 @@ index 1c4b943..91902dc 100644 + from pipenv.utils import chdir + with chdir(ireq.setup_py_dir): + from setuptools.dist import distutils -+ distutils.core.run_setup(ireq.setup_py) ++ dist = distutils.core.run_setup(ireq.setup_py) + except (ImportError, InstallationError, TypeError, AttributeError): + pass + try: -+ dist = ireq.get_dist() ++ dist = ireq.get_dist() if not dist else dist + except InstallationError: + ireq.run_egg_info() + dist = ireq.get_dist() + except (TypeError, ValueError, AttributeError): + pass + else: -+ if dist.has_metadata('requires.txt'): -+ setup_requires = self.finder.get_extras_links( -+ dist.get_metadata_lines('requires.txt') -+ ) ++ setup_requires = getattr(dist, "extras_require", None) ++ if not setup_requires: ++ setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} try: - # Pip < 9 and below + # Pip 9 and below @@ -282,7 +281,7 @@ index 1c4b943..91902dc 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +333,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +332,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -383,7 +382,7 @@ index 1c4b943..91902dc 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -210,6 +435,10 @@ class PyPIRepository(BaseRepository): +@@ -210,6 +434,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -394,7 +393,7 @@ index 1c4b943..91902dc 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -217,24 +446,22 @@ class PyPIRepository(BaseRepository): +@@ -217,24 +445,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. @@ -436,11 +435,11 @@ index 05ec8fd..2f94f6b 100644 +++ b/pipenv/patched/piptools/resolver.py @@ -8,13 +8,14 @@ from itertools import chain, count import os - + from first import first +from pip._vendor.packaging.markers import default_environment from ._compat import InstallRequirement - + from . import click from .cache import DependencyCache from .exceptions import UnsupportedConstraint @@ -448,7 +447,7 @@ index 05ec8fd..2f94f6b 100644 -from .utils import (format_requirement, format_specifier, full_groupby, +from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) - + green = partial(click.style, fg='green') @@ -28,6 +29,7 @@ class RequirementSummary(object): def __init__(self, ireq): @@ -457,11 +456,11 @@ index 05ec8fd..2f94f6b 100644 + self.markers = ireq.markers self.extras = str(sorted(ireq.extras)) self.specifier = str(ireq.specifier) - + @@ -71,7 +73,7 @@ class Resolver(object): with self.repository.allow_all_wheels(): return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs} - + - def resolve(self, max_rounds=10): + def resolve(self, max_rounds=12): """ @@ -517,14 +516,14 @@ index 05ec8fd..2f94f6b 100644 + ireq.extras = ireq.extra elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) - + @@ -283,14 +301,14 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) - self.dependency_cache[ireq] = sorted(str(ireq.req) for ireq in dependencies) + self.dependency_cache[ireq] = sorted(format_requirement(_ireq) for _ireq in dependencies) - + # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] dependency_strings = self.dependency_cache[ireq] log.debug(' {:25} requires {}'.format(format_requirement(ireq), @@ -532,7 +531,7 @@ index 05ec8fd..2f94f6b 100644 for dependency_string in dependency_strings: - yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) + yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) - + def reverse_dependencies(self, ireqs): non_editable = [ireq for ireq in ireqs if not ireq.editable] diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py @@ -555,25 +554,25 @@ index fde5816..23a05f2 100644 @@ -2,6 +2,7 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) - + +import six import os import sys from itertools import chain, groupby @@ -11,13 +12,79 @@ from contextlib import contextmanager from ._compat import InstallRequirement - + from first import first - +from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier +from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from pip._vendor.packaging.markers import Marker, Op, Value, Variable from .click import style - - + + UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} - - + + +def simplify_markers(ireq): + """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" + @@ -643,8 +642,8 @@ index fde5816..23a05f2 100644 if ireq.req is None and ireq.link is not None: @@ -43,16 +110,51 @@ def comment(text): return style(text, fg='green') - - + + -def make_install_requirement(name, version, extras, constraint=False): +def make_install_requirement(name, version, extras, markers, constraint=False): # If no extras are specified, the extras string is blank @@ -652,7 +651,7 @@ index fde5816..23a05f2 100644 if extras: # Sort extras for stability extras_string = "[{}]".format(",".join(sorted(extras))) - + - return InstallRequirement.from_line( - str('{}{}=={}'.format(name, extras_string, version)), - constraint=constraint) @@ -694,8 +693,8 @@ index fde5816..23a05f2 100644 + parts.append("; {0}".format(requirement.marker)) + + return "".join(parts) - - + + def format_requirement(ireq, marker=None): @@ -63,10 +165,10 @@ def format_requirement(ireq, marker=None): if ireq.editable: @@ -703,14 +702,14 @@ index fde5816..23a05f2 100644 else: - line = str(ireq.req).lower() + line = _requirement_to_str_lowercase_name(ireq.req) - + - if marker: - line = '{} ; {}'.format(line, marker) + if marker and ';' not in line: + line = '{}; {}'.format(line, marker) - + return line - + diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index 7e8cdf3..0a0d27d 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index c04dc3f679..9fad7d471a 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -344,7 +344,7 @@ def test_editable_no_args(PipenvInstance): with PipenvInstance() as p: c = p.pipenv("install -e") assert c.return_code != 0 - assert "Please provide path to editable package" in c.err + assert "Error: -e option requires an argument" in c.err @pytest.mark.install diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index c9dded4314..055c39b2e1 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -13,15 +13,10 @@ @pytest.mark.extras @pytest.mark.install @pytest.mark.local -@pytest.mark.parametrize( - "line, pipfile", - [["-e .[dev]", {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}}]], -) -def test_local_extras_install(PipenvInstance, pypi, line, pipfile): +def test_local_extras_install(PipenvInstance, pypi): """Ensure -e .[extras] installs. """ with PipenvInstance(pypi=pypi, chdir=True) as p: - project = Project() setup_py = os.path.join(p.path, "setup.py") with open(setup_py, "w") as fh: contents = """ @@ -40,7 +35,17 @@ def test_local_extras_install(PipenvInstance, pypi, line, pipfile): ) """.strip() fh.write(contents) - project.write_toml({"packages": pipfile, "dev-packages": {}}) + line = "-e .[dev]" + # pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} + project = Project() + with open(os.path.join(p.path, 'Pipfile'), 'w') as fh: + fh.write(""" +[packages] +testpipenv = {path = ".", editable = true, extras = ["dev"]} + +[dev-packages] + """.strip()) + # project.write_toml({"packages": pipfile, "dev-packages": {}}) c = p.pipenv("install") assert c.return_code == 0 assert "testpipenv" in p.lockfile["default"] diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index c51db6ada2..c5915607c3 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -97,16 +97,18 @@ def test_file_urls_work(PipenvInstance, pip_src_dir): @pytest.mark.files @pytest.mark.urls @pytest.mark.needs_internet -def test_local_vcs_urls_work(PipenvInstance, pypi): +def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): + six_dir = tmpdir.join("six") + six_path = Path(six_dir.strpath) with PipenvInstance(pypi=pypi, chdir=True) as p: - six_path = Path(p.path).joinpath("six").absolute() c = delegator.run( - "git clone " "https://github.com/benjaminp/six.git {0}".format(six_path) + "git clone https://github.com/benjaminp/six.git {0}".format(six_dir.strpath) ) assert c.return_code == 0 c = p.pipenv("install git+{0}#egg=six".format(six_path.as_uri())) assert c.return_code == 0 + assert "six" in p.pipfile["packages"] @pytest.mark.e diff --git a/tests/integration/test_sync.py b/tests/integration/test_sync.py index 401eadfe04..c50e259b49 100644 --- a/tests/integration/test_sync.py +++ b/tests/integration/test_sync.py @@ -7,7 +7,7 @@ @pytest.mark.sync def test_sync_error_without_lockfile(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance(pypi=pypi, chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] @@ -40,7 +40,7 @@ def test_mirror_lock_sync(PipenvInstance, pypi): def test_sync_should_not_lock(PipenvInstance, pypi): """Sync should not touch the lock file, even if Pipfile is changed. """ - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance(pypi=pypi, chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages]