diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 7018c898bf..bd2196486b 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -495,15 +495,6 @@ def callback(ctx, param, value): def setup_verbosity(ctx, param, value): if not value: return - import logging - - loggers = ("pip",) - if value == 1: - for logger in loggers: - logging.getLogger(logger).setLevel(logging.INFO) - elif value == -1: - for logger in loggers: - logging.getLogger(logger).setLevel(logging.CRITICAL) ctx.ensure_object(State).project.s.PIPENV_VERBOSITY = value diff --git a/pipenv/core.py b/pipenv/core.py index 5f3adadf57..1531ab155e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2,6 +2,7 @@ import logging import os import shutil +import subprocess import sys import tempfile import time @@ -10,12 +11,9 @@ from posixpath import expandvars from typing import Dict, List, Optional, Union -import dotenv -import pipfile -import vistir - -from pipenv import environments, exceptions, pep508checker, progress +from pipenv import environments, exceptions, pep508checker from pipenv._compat import decode_for_output, fix_utf8 +from pipenv.patched import pipfile from pipenv.patched.pip._internal.build_env import _get_runnable_pip from pipenv.patched.pip._internal.exceptions import PipError from pipenv.patched.pip._internal.network.session import PipSession @@ -49,7 +47,7 @@ system_which, ) from pipenv.utils.spinner import create_spinner -from pipenv.vendor import click +from pipenv.vendor import click, dotenv, vistir from pipenv.vendor.requirementslib.models.requirements import Requirement if MYPY_RUNNING: @@ -652,53 +650,53 @@ def _cleanup_procs(project, procs, failed_deps_queue, retry=True): except AttributeError: out, err = c.stdout, c.stderr failed = c.returncode != 0 - if "Ignoring" in out: - click.secho(out.strip(), fg="yellow") - elif project.s.is_verbose(): - click.secho(out.strip() or err.strip(), fg="cyan") + if project.s.is_verbose(): + click.secho(out.strip() or err.strip(), fg="yellow") # The Installation failed... if failed: - # If there is a mismatch in installed locations or the install fails - # due to wrongful disabling of pep517, we should allow for - # additional passes at installation - if "does not match installed location" in err: - project.environment.expand_egg_links() - click.echo( - "{}".format( - click.style( - "Failed initial installation: Failed to overwrite existing " - "package, likely due to path aliasing. Expanding and trying " - "again!", - fg="yellow", + deps = getattr(c, "deps", {}).copy() + for dep in deps: + # If there is a mismatch in installed locations or the install fails + # due to wrongful disabling of pep517, we should allow for + # additional passes at installation + if "does not match installed location" in err: + project.environment.expand_egg_links() + click.echo( + "{}".format( + click.style( + "Failed initial installation: Failed to overwrite existing " + "package, likely due to path aliasing. Expanding and trying " + "again!", + fg="yellow", + ) ) ) - ) - dep = c.dep.copy() - dep.use_pep517 = True - elif "Disabling PEP 517 processing is invalid" in err: - dep = c.dep.copy() - dep.use_pep517 = True - elif not retry: - # The Installation failed... - # We echo both c.stdout and c.stderr because pip returns error details on out. - err = err.strip().splitlines() if err else [] - out = out.strip().splitlines() if out else [] - err_lines = [line for message in [out, err] for line in message] - # Return the subprocess' return code. - raise exceptions.InstallError(c.dep.name, extra=err_lines) - else: - # Alert the user. - dep = c.dep.copy() - dep.use_pep517 = False - click.echo( - "{} {}! Will try again.".format( - click.style("An error occurred while installing", fg="red"), - click.style(dep.as_line(), fg="green"), - ), - err=True, - ) - # Save the Failed Dependency for later. - failed_deps_queue.put(dep) + if dep: + dep.use_pep517 = True + elif "Disabling PEP 517 processing is invalid" in err: + if dep: + dep.use_pep517 = True + elif not retry: + # The Installation failed... + # We echo both c.stdout and c.stderr because pip returns error details on out. + err = err.strip().splitlines() if err else [] + out = out.strip().splitlines() if out else [] + err_lines = [line for message in [out, err] for line in message] + # Return the subprocess' return code. + raise exceptions.InstallError(deps, extra=err_lines) + else: + # Alert the user. + if dep: + dep.use_pep517 = False + click.echo( + "{} {}! Will try again.".format( + click.style("An error occurred while installing", fg="red"), + click.style(dep.as_line() if dep else "", fg="green"), + ), + err=True, + ) + # Save the Failed Dependency for later. + failed_deps_queue.put(dep) def batch_install( @@ -710,7 +708,6 @@ def batch_install( no_deps=True, ignore_hashes=False, allow_global=False, - blocking=False, pypi_mirror=None, retry=True, sequential_deps=None, @@ -722,32 +719,19 @@ def batch_install( if sequential_deps is None: sequential_deps = [] failed = not retry - if not failed: - label = INSTALL_LABEL if not environments.PIPENV_HIDE_EMOJIS else "" - else: - label = INSTALL_LABEL2 - deps_to_install = deps_list[:] deps_to_install.extend(sequential_deps) deps_to_install = [ dep for dep in deps_to_install if not project.environment.is_satisfied(dep) ] - sequential_dep_names = [d.name for d in sequential_deps] - - deps_list_bar = progress.bar( - deps_to_install, width=32, label=label, hide=environments.PIPENV_IS_CI - ) - trusted_hosts = [] - # Install these because - for dep in deps_list_bar: - extra_indexes = [] + is_artifact = False + for dep in deps_to_install: if dep.req.req: dep.req.req = strip_extras_markers_from_requirement(dep.req.req) if dep.markers: dep.markers = str(strip_extras_markers_from_requirement(dep.get_markers())) # Install the module. - is_artifact = False if dep.is_file_or_url and ( dep.is_direct_url or any(dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"]) @@ -755,44 +739,34 @@ def batch_install( is_artifact = True elif dep.is_vcs: is_artifact = True - if not project.s.PIPENV_RESOLVE_VCS and is_artifact and not dep.editable: - skip_dependencies = False - else: - skip_dependencies = no_deps - - with vistir.contextmanagers.temp_environ(): - if not allow_global: - os.environ["PIP_USER"] = "0" - if "PYTHONHOME" in os.environ: - del os.environ["PYTHONHOME"] - if "GIT_CONFIG" in os.environ and dep.is_vcs: - del os.environ["GIT_CONFIG"] - use_pep517 = True - if failed and not dep.is_vcs: - use_pep517 = getattr(dep, "use_pep517", False) - - is_sequential = sequential_deps and dep.name in sequential_dep_names - is_blocking = any([dep.editable, dep.is_vcs, blocking, is_sequential]) - c = pip_install( - project, - dep, - ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), - allow_global=allow_global, - no_deps=skip_dependencies, - block=is_blocking, - index=dep.index, - requirements_dir=requirements_dir, - pypi_mirror=pypi_mirror, - trusted_hosts=trusted_hosts, - extra_indexes=extra_indexes, - use_pep517=use_pep517, - use_constraint=False, # no need to use constraints, it's written in lockfile - ) - c.dep = dep + with vistir.contextmanagers.temp_environ(): + if not allow_global: + os.environ["PIP_USER"] = "0" + if "PYTHONHOME" in os.environ: + del os.environ["PYTHONHOME"] + if "GIT_CONFIG" in os.environ: + del os.environ["GIT_CONFIG"] + use_pep517 = True + if failed and not is_artifact: + use_pep517 = False + + cmds = pip_install_deps( + project, + deps=deps_to_install, + allow_global=allow_global, + ignore_hashes=ignore_hashes, + no_deps=no_deps, + requirements_dir=requirements_dir, + pypi_mirror=pypi_mirror, + trusted_hosts=trusted_hosts, + use_pep517=use_pep517, + use_constraint=False, # no need to use constraints, it's written in lockfile + ) + + for c in cmds: procs.put(c) - if procs.full() or procs.qsize() == len(deps_list) or is_sequential: - _cleanup_procs(project, procs, failed_deps_queue, retry=retry) + _cleanup_procs(project, procs, failed_deps_queue, retry=retry) def do_install_dependencies( @@ -853,7 +827,6 @@ def do_install_dependencies( "no_deps": not skip_lock, "ignore_hashes": ignore_hashes, "allow_global": allow_global, - "blocking": not concurrent, "pypi_mirror": pypi_mirror, "sequential_deps": editable_or_vcs_deps, } @@ -1377,7 +1350,7 @@ def get_pip_args( ], "src_dir": src_dir, } - arg_set = [] + arg_set = ["--no-input"] for key in arg_map.keys(): if key in locals() and locals().get(key): arg_set.extend(arg_map.get(key)) @@ -1414,7 +1387,6 @@ def write_requirement_to_file( project: Project, requirement: Requirement, requirements_dir: Optional[str] = None, - src_dir: Optional[str] = None, include_hashes: bool = True, ) -> str: if not requirements_dir: @@ -1495,7 +1467,6 @@ def pip_install( project, requirement, requirements_dir=requirements_dir, - src_dir=src_dir, include_hashes=not ignore_hashes, ) sources = get_source_list( @@ -1572,6 +1543,158 @@ def pip_install( return c +def pip_install_deps( + project, + deps=None, + allow_global=False, + ignore_hashes=False, + no_deps=False, + pre=False, + dev=False, + selective_upgrade=False, + requirements_dir=None, + pypi_mirror=None, + trusted_hosts=None, + use_pep517=True, + use_constraint=False, +): + if not trusted_hosts: + trusted_hosts = [] + trusted_hosts.extend(os.environ.get("PIP_TRUSTED_HOSTS", [])) + if not allow_global: + src_dir = os.getenv( + "PIP_SRC", os.getenv("PIP_SRC_DIR", project.virtualenv_src_location) + ) + else: + src_dir = os.getenv("PIP_SRC", os.getenv("PIP_SRC_DIR")) + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir( + prefix="pipenv", suffix="requirements" + ) + + standard_requirements = tempfile.NamedTemporaryFile( + prefix="pipenv-", suffix="-hashed-reqs.txt", dir=requirements_dir, delete=False + ) + editable_requirements = tempfile.NamedTemporaryFile( + prefix="pipenv-", suffix="-reqs.txt", dir=requirements_dir, delete=False + ) + for requirement in deps: + ignore_hash = ignore_hashes + vcs_or_editable = requirement.is_vcs or requirement.vcs or requirement.editable + if vcs_or_editable: + ignore_hash = True + if requirement and vcs_or_editable: + requirement.index = None + + line = requirement.line_instance.get_line( + with_prefix=True, + with_hashes=not ignore_hash, + with_markers=True, + as_list=False, + ) + if project.s.is_verbose(): + click.echo( + f"Writing supplied requirement line to temporary file: {line!r}", err=True + ) + target = editable_requirements if vcs_or_editable else standard_requirements + target.write(vistir.misc.to_bytes(line)) + target.write(vistir.misc.to_bytes("\n")) + standard_requirements.close() + editable_requirements.close() + + cmds = [] + files = [] + standard_deps = list(filter(lambda d: not (d.is_vcs or d.vcs or d.editable), deps)) + if standard_deps: + files.append(standard_requirements) + editable_deps = list(filter(lambda d: d.is_vcs or d.vcs or d.editable, deps)) + if editable_deps: + files.append(editable_requirements) + for file in files: + pip_command = [ + project_python(project, system=allow_global), + _get_runnable_pip(), + "install", + ] + pip_args = get_pip_args( + project, + pre=pre, + verbose=False, # When True, the subprocess fails to recognize the EOF when reading stdout. + upgrade=True, + selective_upgrade=selective_upgrade, + no_use_pep517=not use_pep517, + no_deps=no_deps, + ) + sources = get_source_list( + project, + index=None, + extra_indexes=None, + trusted_hosts=trusted_hosts, + pypi_mirror=pypi_mirror, + ) + pip_command.extend(prepare_pip_source_args(sources)) + pip_command.extend(pip_args) + pip_command.extend(["-r", normalize_path(file.name)]) + if dev and use_constraint: + default_constraints = get_constraints_from_deps(project.packages) + constraint_filename = prepare_constraint_file( + default_constraints, + directory=requirements_dir, + sources=None, + pip_args=None, + ) + pip_command.extend(["-c", normalize_path(constraint_filename)]) + if project.s.is_verbose(): + msg = f"Install Phase: {'Standard Requirements' if file == standard_requirements else 'Editable Requirements'}" + click.echo( + click.style(msg, bold=True), + err=True, + ) + for requirement in ( + standard_deps if file == standard_requirements else editable_deps + ): + click.echo( + click.style( + f"Preparing Installation of {requirement.name!r}", bold=True + ), + err=True, + ) + click.secho(f"$ {cmd_list_to_shell(pip_command)}", fg="cyan", err=True) + cache_dir = Path(project.s.PIPENV_CACHE_DIR) + default_exists_action = "w" + if selective_upgrade: + default_exists_action = "i" + exists_action = project.s.PIP_EXISTS_ACTION or default_exists_action + pip_config = { + "PIP_CACHE_DIR": cache_dir.as_posix(), + "PIP_WHEEL_DIR": cache_dir.joinpath("wheels").as_posix(), + "PIP_DESTINATION_DIR": cache_dir.joinpath("pkgs").as_posix(), + "PIP_EXISTS_ACTION": exists_action, + "PATH": os.environ.get("PATH"), + } + if src_dir: + if project.s.is_verbose(): + click.echo(f"Using source directory: {src_dir!r}", err=True) + pip_config.update({"PIP_SRC": src_dir}) + c = subprocess_run(pip_command, block=False, capture_output=True, env=pip_config) + if file == standard_requirements: + c.deps = standard_deps + else: + c.deps = editable_deps + c.env = pip_config + cmds.append(c) + if project.s.is_verbose(): + while True: + line = c.stdout.readline() + if line == "": + break + if "Ignoring" in line: + click.secho(line, fg="red", err=True) + elif line: + click.secho(line, fg="yellow", err=True) + return cmds + + def pip_download(project, package_name): cache_dir = Path(project.s.PIPENV_CACHE_DIR) pip_config = { @@ -2499,8 +2622,6 @@ def inline_activate_virtual_environment(project): def _launch_windows_subprocess(script, env): - import subprocess - path = env.get("PATH", "") command = system_which(script.command, path=path) diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index f0741f15f2..3c7959cb4f 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -316,13 +316,10 @@ def test_skip_requirements_when_pipfile(PipenvInstance): contents = """ [packages] six = "*" -fake_package = "<0.12" """.strip() f.write(contents) c = p.pipenv("install") assert c.returncode == 0 - assert "fake_package" in p.pipfile["packages"] - assert "fake-package" in p.lockfile["default"] assert "six" in p.pipfile["packages"] assert "six" in p.lockfile["default"] assert "requests" not in p.pipfile["packages"] diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index a3ee6456b5..85fb99e809 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -23,7 +23,6 @@ def test_package_environment_markers(PipenvInstance): c = p.pipenv('install') assert c.returncode == 0 - assert 'Ignoring' in c.stdout assert 'markers' in p.lockfile['default']['fake-package'], p.lockfile["default"] c = p.pipenv('run python -c "import fake_package;"') @@ -37,14 +36,7 @@ def test_platform_python_implementation_marker(PipenvInstance): incorrectly. """ with PipenvInstance() as p: - with open(p.pipfile_path, 'w') as f: - contents = """ -[packages] -depends-on-marked-package = "*" - """.strip() - f.write(contents) - - c = p.pipenv('install') + c = p.pipenv('install depends-on-marked-package') assert c.returncode == 0 # depends-on-marked-package has an install_requires of @@ -71,8 +63,6 @@ def test_specific_package_environment_markers(PipenvInstance): c = p.pipenv('install') assert c.returncode == 0 - - assert 'Ignoring' in c.stdout assert 'markers' in p.lockfile['default']['fake-package'] c = p.pipenv('run python -c "import fake_package;"') diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 28e7f21f43..4a7861e1d5 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -376,13 +376,11 @@ def test_private_index_lock_requirements(PipenvInstance_NoPyPI): @pytest.mark.install # private indexes need to be uncached for resolution @pytest.mark.requirements @pytest.mark.needs_internet -def test_private_index_mirror_lock_requirements(PipenvInstance_NoPyPI): +def test_private_index_lock_requirements(PipenvInstance_NoPyPI): # Don't use the local fake pypi with temp_environ(), PipenvInstance_NoPyPI(chdir=True) as p: # Using pypi.python.org as pipenv-test-public-package is not # included in the local pypi mirror - mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz.org/simple") - # os.environ.pop('PIPENV_TEST_INDEX', None) with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -397,12 +395,13 @@ def test_private_index_mirror_lock_requirements(PipenvInstance_NoPyPI): [packages] six = {version = "*", index = "testpypi"} -fake-package = "*" +pipenv-test-public-package = "*" """.strip() f.write(contents) - c = p.pipenv(f'install -v --pypi-mirror {mirror_url}') + c = p.pipenv(f'install -v') assert c.returncode == 0 + @pytest.mark.lock @pytest.mark.install @pytest.mark.skip_windows diff --git a/tests/integration/test_sync.py b/tests/integration/test_sync.py index d5683db0bf..b80f4b4850 100644 --- a/tests/integration/test_sync.py +++ b/tests/integration/test_sync.py @@ -94,25 +94,6 @@ def test_sync_sequential_detect_errors(PipenvInstance): assert c.returncode != 0 -@pytest.mark.sync -@pytest.mark.lock -def test_sync_sequential_verbose(PipenvInstance): - with PipenvInstance() as p: - with open(p.pipfile_path, 'w') as f: - contents = """ -[packages] -requests = "*" - """.strip() - f.write(contents) - - c = p.pipenv('lock') - assert c.returncode == 0 - - c = p.pipenv('sync --sequential --verbose') - for package in p.lockfile['default']: - assert f'Successfully installed {package}' in c.stdout - - @pytest.mark.sync def test_sync_consider_pip_target(PipenvInstance): """