From dcfce70817376ebf0ef5fe9a0f45ca3a42db81ff Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 30 Nov 2018 02:22:59 -0500 Subject: [PATCH 01/17] Fix python path discovery if not called `python` - Begin a refactor of `delegator.run` invocation to ensure we capture and handle failures with our own exception wrappers - Additoinally capture output and error logging and command information when running in verbose mode (should avoid significant repitition in the codebase) - Refactor `which` and `system_which` to fallback to pythonfinder's implementation - Abstract `is_python_command` to identify whether we are looking for python, this enables us to rely on `pythonfinder.Finder.find_all_python_versions()` to ensure we aren't skipping python versions - Fixes #2783 Signed-off-by: Dan Ryan --- news/2783.bugfix.rst | 2 + pipenv/cmdparse.py | 2 +- pipenv/core.py | 193 ++++++++++++++++++++++++--------------- pipenv/exceptions.py | 89 ++++++++++++++---- pipenv/utils.py | 148 ++++++++++++++++++++---------- tests/unit/test_utils.py | 23 +++++ 6 files changed, 313 insertions(+), 144 deletions(-) create mode 100644 news/2783.bugfix.rst diff --git a/news/2783.bugfix.rst b/news/2783.bugfix.rst new file mode 100644 index 0000000000..7fa3cfd1f3 --- /dev/null +++ b/news/2783.bugfix.rst @@ -0,0 +1,2 @@ +Fixed an issue which caused errors due to reliance on the system utilities ``which`` and ``where`` which may not always exist on some systems. +- Fixed a bug which caused periodic failures in python discovery when executables named ``python`` were not present on the target ``$PATH``. diff --git a/pipenv/cmdparse.py b/pipenv/cmdparse.py index 21aba77e4f..2e550e2220 100644 --- a/pipenv/cmdparse.py +++ b/pipenv/cmdparse.py @@ -10,7 +10,7 @@ class ScriptEmptyError(ValueError): def _quote_if_contains(value, pattern): - if next(re.finditer(pattern, value), None): + if next(iter(re.finditer(pattern, value)), None): return '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', value)) return value diff --git a/pipenv/core.py b/pipenv/core.py index b1f8c9fe4d..2f88a5304f 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -34,7 +34,8 @@ escape_cmd, escape_grouped_arguments, find_windows_executable, get_canonical_names, is_pinned, is_pypi_url, is_required_version, is_star, is_valid_url, parse_indexes, pep423_name, prepare_pip_source_args, - proper_case, python_version, venv_resolve_deps + proper_case, python_version, venv_resolve_deps, run_command, + is_python_command, find_python ) @@ -86,16 +87,19 @@ def which(command, location=None, allow_global=False): location = os.environ.get("VIRTUAL_ENV", None) if not (location and os.path.exists(location)) and not allow_global: raise RuntimeError("location not created nor specified") + + version_str = "python{0}".format(".".join([str(v) for v in sys.version_info[:2]])) + is_python = command in ("python", os.path.basename(sys.executable), version_str) if not allow_global: if os.name == "nt": p = find_windows_executable(os.path.join(location, "Scripts"), command) else: p = os.path.join(location, "bin", command) else: - if command == "python": + if is_python: p = sys.executable if not os.path.exists(p): - if command == "python": + if is_python: p = sys.executable or system_which("python") else: p = system_which(command) @@ -323,26 +327,18 @@ def find_a_system_python(line): * Search for "python" and "pythonX.Y" executables in PATH to find a match. * Nothing fits, return None. """ - if not line: - return None + if os.path.isabs(line): return line from .vendor.pythonfinder import Finder - finder = Finder(system=False, global_search=True) + if not line: + return next(iter(finder.find_all_python_versions()), None) + # Use the windows finder executable if (line.startswith("py ") or line.startswith("py.exe ")) and os.name == "nt": line = line.split(" ", 1)[1].lstrip("-") - elif line.startswith("py"): - python_entry = finder.which(line) - if python_entry: - return python_entry.path.as_posix() - return None - python_entry = finder.find_python_version(line) - if not python_entry: - python_entry = finder.which("python{0}".format(line)) - if python_entry: - return python_entry.path.as_posix() - return None + python_entry = find_python(finder, line) + return python_entry def ensure_python(three=None, python=None): @@ -1433,8 +1429,8 @@ def pip_install( if "--hash" not in f.read(): ignore_hashes = True else: - ignore_hashes = True - install_reqs = requirement.as_line(as_list=True, include_hashes=not ignore_hashes) + ignore_hashes = True if not requirement.hashes else False + install_reqs = requirement.as_line(as_list=True) if not requirement.markers: install_reqs = [escape_cmd(r) for r in install_reqs] elif len(install_reqs) > 1: @@ -1507,18 +1503,59 @@ def pip_download(package_name): return c +def fallback_which(command, location=None, allow_global=False, system=False): + """ + A fallback implementation of the `which` utility command that relies exclusively on + searching the path for commands. + + :param str command: The command to search for, optional + :param str location: The search location to prioritize (prepend to path), defaults to None + :param bool allow_global: Whether to search the global path, defaults to False + :param bool system: Whether to use the system python instead of pipenv's python, defaults to False + :raises ValueError: Raised if no command is provided + :raises TypeError: Raised if the command provided is not a string + :return: A path to the discovered command location + :rtype: str + """ + + from .vendor.pythonfinder import Finder + if not command: + raise ValueError("fallback_which: Must provide a command to search for...") + if not isinstance(command, six.string_types): + raise TypeError("Provided command must be a string, received {0!r}".format(command)) + global_search = system or allow_global + finder = Finder(system=False, global_search=global_search, path=location) + if is_python_command(command): + result = find_python(finder, command) + if result: + return result + result = finder.which(command) + if result: + return result.path.as_posix() + return "" + + def which_pip(allow_global=False): """Returns the location of virtualenv-installed pip.""" + + location = None + if "VIRTUAL_ENV" in os.environ: + location = os.environ["VIRTUAL_ENV"] if allow_global: - if "VIRTUAL_ENV" in os.environ: - return which("pip", location=os.environ["VIRTUAL_ENV"]) + if location: + pip = which("pip", location=location) + if pip: + return pip for p in ("pip", "pip3", "pip2"): where = system_which(p) if where: return where - return which("pip") + pip = which("pip") + if not pip: + pip = fallback_which("pip", allow_global=allow_global, location=location) + return pip def system_which(command, mult=False): @@ -1528,6 +1565,7 @@ def system_which(command, mult=False): vistir.compat.fs_str(k): vistir.compat.fs_str(val) for k, val in os.environ.items() } + result = None try: c = delegator.run("{0} {1}".format(_which, command)) try: @@ -1542,21 +1580,20 @@ def system_which(command, mult=False): ) assert c.return_code == 0 except AssertionError: - return None if not mult else [] + result = fallback_which(command, allow_global=True) except TypeError: - from .vendor.pythonfinder import Finder - finder = Finder() - result = finder.which(command) - if result: - return result.path.as_posix() - return + if not result: + result = fallback_which(command, allow_global=True) else: - result = c.out.strip() or c.err.strip() - if mult: - return result.split("\n") + if not result: + result = next(iter([c.out, c.err]), "").split("\n") + result = next(iter(result)) if not mult else result + return result + if not result: + result = fallback_which(command, allow_global=True) + result = [result] if mult else result + return result - else: - return result.split("\n")[0] def format_help(help): @@ -2173,6 +2210,7 @@ def do_uninstall( p for normalized, p in selected_pkg_map.items() if normalized in (used_packages - bad_pkgs) ] + pip_path = None for normalized, package_name in selected_pkg_map.items(): click.echo( crayons.white( @@ -2182,12 +2220,10 @@ def do_uninstall( # Uninstall the package. if package_name in packages_to_remove: with project.environment.activated(): - cmd = "{0} uninstall {1} -y".format( - escape_grouped_arguments(which_pip(allow_global=system)), package_name, - ) - if environments.is_verbose(): - click.echo("$ {0}".format(cmd)) - c = delegator.run(cmd) + if pip_path is None: + pip_path = which_pip(allow_global=system) + cmd = [pip_path, "uninstall", package_name, "-y"] + c = run_command(cmd) click.echo(crayons.blue(c.out)) if c.return_code != 0: failure = True @@ -2441,6 +2477,7 @@ def do_check( args=None, pypi_mirror=None, ): + from pipenv.vendor.vistir.compat import JSONDecodeError if not system: # Ensure that virtualenv is available. ensure_project( @@ -2471,18 +2508,27 @@ def do_check( sys.exit(1) else: sys.exit(0) - click.echo(crayons.normal(fix_utf8("Checking PEP 508 requirements…"), bold=True)) - if system: - python = system_which("python") - else: + click.echo(crayons.normal(decode_for_output("Checking PEP 508 requirements…"), bold=True)) + pep508checker_path = pep508checker.__file__.rstrip("cdo") + safety_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "patched", "safety.zip" + ) + if not system: python = which("python") + else: + python = system_which("python") + _cmd = [python,] # Run the PEP 508 checker in the virtualenv. - c = delegator.run( - '"{0}" {1}'.format( - python, escape_grouped_arguments(pep508checker.__file__.rstrip("cdo")) - ) - ) - results = simplejson.loads(c.out) + cmd = _cmd + [pep508checker_path] + c = run_command(cmd) + try: + results = simplejson.loads(c.out.strip()) + except JSONDecodeError: + click.echo("{0}\n{1}".format( + crayons.white(decode_for_output("Failed parsing pep508 results: "), bold=True), + c.out.strip() + )) + sys.exit(1) # Load the pipfile. p = pipfile.Pipfile.load(project.pipfile_location) failed = False @@ -2507,13 +2553,9 @@ def do_check( sys.exit(1) else: click.echo(crayons.green("Passed!")) - click.echo(crayons.normal(fix_utf8("Checking installed package safety…"), bold=True)) - path = pep508checker.__file__.rstrip("cdo") - path = os.sep.join(__file__.split(os.sep)[:-1] + ["patched", "safety.zip"]) - if not system: - python = which("python") - else: - python = system_which("python") + click.echo(crayons.normal( + decode_for_output("Checking installed package safety…"), bold=True) + ) if ignore: ignored = "--ignore {0}".format(" --ignore ".join(ignore)) click.echo( @@ -2524,17 +2566,13 @@ def do_check( ) else: ignored = "" - c = delegator.run( - '"{0}" {1} check --json --key={2} {3}'.format( - python, escape_grouped_arguments(path), PIPENV_PYUP_API_KEY, ignored - ) - ) + key = "--key={0}".format(PIPENV_PYUP_API_KEY) + cmd = _cmd + [safety_path, "check", "--json", key, ignored] + c = run_command(cmd) try: results = simplejson.loads(c.out) - except ValueError: - click.echo("An error occurred:", err=True) - click.echo(c.err if len(c.err) > 0 else c.out, err=True) - sys.exit(1) + except (ValueError, JSONDecodeError): + raise exceptions.JSONParseError(c.out, c.err) for (package, resolved, installed, description, vuln) in results: click.echo( "{0}: {1} {2} resolved ({3} installed)!".format( @@ -2553,7 +2591,9 @@ def do_check( def do_graph(bare=False, json=False, json_tree=False, reverse=False): + from pipenv.vendor.vistir.compat import JSONDecodeError import pipdeptree + pipdeptree_path = pipdeptree.__file__.rstrip("cdo") try: python_path = which("python") except AttributeError: @@ -2618,11 +2658,9 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): err=True, ) sys.exit(1) - cmd = '"{0}" {1} {2} -l'.format( - python_path, escape_grouped_arguments(pipdeptree.__file__.rstrip("cdo")), flag - ) + cmd_args = [python_path, pipdeptree_path, flag, "-l"] + c = run_command(cmd_args) # Run dep-tree. - c = delegator.run(cmd) if not bare: if json: data = [] @@ -2644,9 +2682,14 @@ def traverse(obj): obj["dependencies"] = traverse(obj["dependencies"]) return obj - data = traverse(simplejson.loads(c.out)) - click.echo(simplejson.dumps(data, indent=4)) - sys.exit(0) + try: + parsed = simplejson.loads(c.out.strip()) + except JSONDecodeError: + raise exceptions.JSONParseError(c.out, c.err) + else: + data = traverse(parsed) + click.echo(simplejson.dumps(data, indent=4)) + sys.exit(0) else: for line in c.out.strip().split("\n"): # Ignore bad packages as top level. @@ -2755,8 +2798,8 @@ def do_clean( ) ) # Uninstall the package. - cmd_str = Script.parse(cmd + [apparent_bad_package]).cmdify() - c = delegator.run(cmd_str, block=True) + cmd = [which_pip(), "uninstall", apparent_bad_package, "-y"] + c = run_command(cmd) if c.return_code != 0: failure = True sys.exit(int(failure)) diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 47dfc718a4..725f382ec9 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -9,7 +9,7 @@ import six from . import environments -from ._compat import fix_utf8 +from ._compat import decode_for_output from .patched import crayons from .vendor.click._compat import get_text_stderr from .vendor.click.exceptions import ( @@ -36,7 +36,7 @@ def handle_exception(exc_type, exception, traceback, hook=sys.excepthook): line = "[pipenv.exceptions.{0!s}]: {1}".format( exception.__class__.__name__, line ) - click_echo(fix_utf8(line), err=True) + click_echo(decode_for_output(line), err=True) exception.show() @@ -64,8 +64,57 @@ def show(self, file=None): extra = "[pipenv.exceptions.{0!s}]: {1}".format( self.__class__.__name__, extra ) + extra = decode_for_output(extra, file) click_echo(extra, file=file) - click_echo(fix_utf8("{0}".format(self.message)), file=file) + click_echo(decode_for_output("{0}".format(self.message), file), file=file) + + +class PipenvCmdError(PipenvException): + def __init__(self, cmd, out="", err="", exit_code=1): + self.cmd = cmd + self.out = out + self.err = err + self.exit_code = exit_code + message = "Error running command: {0}".format(cmd) + PipenvException.__init__(self, message) + + def show(self, file=None): + if file is None: + file = get_text_stderr() + click_echo("{0} {1}".format( + crayons.red("Error running command: "), + crayons.white(decode_for_output("$ {0}".format(self.cmd), file), bold=True) + ), err=True) + if self.out: + click_echo("{0} {1}".format( + crayons.white("OUTPUT: "), + decode_for_output(self.out, file) + ), err=True) + if self.err: + click_echo("{0} {1}".format( + crayons.white("STDERR: "), + decode_for_output(self.err, file) + ), err=True) + + +class JSONParseError(PipenvException): + def __init__(self, contents="", error_text=""): + self.error_text = error_text + PipenvException.__init__(self, contents) + + def show(self, file=None): + if file is None: + file = get_text_stderr() + message = "{0}\n{1}".format( + crayons.white("Failed parsing JSON results:", bold=True), + decode_for_output(self.message.strip(), file) + ) + click_echo(self.message, err=True) + if self.error_text: + click_echo("{0} {1}".format( + crayons.white("ERROR TEXT:", bold=True), + decode_for_output(self.error_text, file) + ), err=True) class PipenvUsageError(UsageError): @@ -78,7 +127,7 @@ def __init__(self, message=None, ctx=None, **kwargs): message = formatted_message.format(msg_prefix, crayons.white(message, bold=True)) self.message = message extra = kwargs.pop("extra", []) - UsageError.__init__(self, fix_utf8(message), ctx) + UsageError.__init__(self, decode_for_output(message), ctx) self.extra = extra def show(self, file=None): @@ -93,7 +142,7 @@ def show(self, file=None): for extra in self.extra: if color: extra = getattr(crayons, color, "blue")(extra) - click_echo(fix_utf8(extra), file=file) + click_echo(decode_for_output(extra, file), file=file) hint = '' if (self.cmd is not None and self.cmd.get_help_option(self.ctx) is not None): @@ -117,7 +166,7 @@ def __init__(self, filename, message=None, **kwargs): crayons.white("{0} not found!".format(filename), bold=True), message ) - FileError.__init__(self, filename=filename, hint=fix_utf8(message), **kwargs) + FileError.__init__(self, filename=filename, hint=decode_for_output(message), **kwargs) self.extra = extra def show(self, file=None): @@ -127,7 +176,7 @@ def show(self, file=None): if isinstance(self.extra, six.string_types): self.extra = [self.extra,] for extra in self.extra: - click_echo(fix_utf8(extra), file=file) + click_echo(decode_for_output(extra, file), file=file) click_echo(self.message, file=file) @@ -137,10 +186,10 @@ def __init__(self, filename="Pipfile", extra=None, **kwargs): message = ("{0} {1}".format( crayons.red("Aborting!", bold=True), crayons.white("Please ensure that the file exists and is located in your" - " project root directory.", bold=True) + " project root directory.", bold=True) ) ) - super(PipfileNotFound, self).__init__(filename, message=fix_utf8(message), extra=extra, **kwargs) + super(PipfileNotFound, self).__init__(filename, message=decode_for_output(message), extra=extra, **kwargs) class LockfileNotFound(PipenvFileError): @@ -151,7 +200,7 @@ def __init__(self, filename="Pipfile.lock", extra=None, **kwargs): crayons.red("$ pipenv lock", bold=True), crayons.white("before you can continue.", bold=True) ) - super(LockfileNotFound, self).__init__(filename, message=fix_utf8(message), extra=extra, **kwargs) + super(LockfileNotFound, self).__init__(filename, message=decode_for_output(message), extra=extra, **kwargs) class DeployException(PipenvUsageError): @@ -159,13 +208,13 @@ def __init__(self, message=None, **kwargs): if not message: message = crayons.normal("Aborting deploy", bold=True) extra = kwargs.pop("extra", []) - PipenvUsageError.__init__(self, message=fix_utf8(message), extra=extra, **kwargs) + PipenvUsageError.__init__(self, message=decode_for_output(message), extra=extra, **kwargs) class PipenvOptionsError(PipenvUsageError): def __init__(self, option_name, message=None, ctx=None, **kwargs): extra = kwargs.pop("extra", []) - PipenvUsageError.__init__(self, message=fix_utf8(message), ctx=ctx, **kwargs) + PipenvUsageError.__init__(self, message=decode_for_output(message), ctx=ctx, **kwargs) self.extra = extra self.option_name = option_name @@ -192,7 +241,7 @@ def __init__(self, hint=None, **kwargs): hint = "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), hint) filename = project.pipfile_location extra = kwargs.pop("extra", []) - PipenvFileError.__init__(self, filename, fix_utf8(hint), extra=extra, **kwargs) + PipenvFileError.__init__(self, filename, decode_for_output(hint), extra=extra, **kwargs) class SetupException(PipenvException): @@ -208,7 +257,7 @@ def __init__(self, message=None, **kwargs): "There was an unexpected error while activating your virtualenv. " "Continuing anyway..." ) - PipenvException.__init__(self, fix_utf8(message), **kwargs) + PipenvException.__init__(self, decode_for_output(message), **kwargs) class VirtualenvActivationException(VirtualenvException): @@ -219,7 +268,7 @@ def __init__(self, message=None, **kwargs): "not activated. Continuing anyway…" ) self.message = message - VirtualenvException.__init__(self, fix_utf8(message), **kwargs) + VirtualenvException.__init__(self, decode_for_output(message), **kwargs) class VirtualenvCreationException(VirtualenvException): @@ -227,7 +276,7 @@ def __init__(self, message=None, **kwargs): if not message: message = "Failed to create virtual environment." self.message = message - VirtualenvException.__init__(self, fix_utf8(message), **kwargs) + VirtualenvException.__init__(self, decode_for_output(message), **kwargs) class UninstallError(PipenvException): @@ -243,7 +292,7 @@ def __init__(self, package, command, return_values, return_code, **kwargs): crayons.yellow(str(package), bold=True) ) self.exit_code = return_code - PipenvException.__init__(self, message=fix_utf8(message), extra=extra) + PipenvException.__init__(self, message=decode_for_output(message), extra=extra) self.extra = extra @@ -260,7 +309,7 @@ def __init__(self, package, **kwargs): crayons.yellow("Package installation failed...") ) extra = kwargs.pop("extra", []) - PipenvException.__init__(self, message=fix_utf8(message), extra=extra, **kwargs) + PipenvException.__init__(self, message=decode_for_output(message), extra=extra, **kwargs) class CacheError(PipenvException): @@ -271,7 +320,7 @@ def __init__(self, path, **kwargs): crayons.white(path), crayons.white('Consider trying "pipenv lock --clear" to clear the cache.') ) - super(PipenvException, self).__init__(message=fix_utf8(message)) + PipenvException.__init__(self, message=decode_for_output(message)) class ResolutionFailure(PipenvException): @@ -304,4 +353,4 @@ def __init__(self, message, no_version_found=False): "See PEP440 for more information." ) ) - super(ResolutionFailure, self).__init__(fix_utf8(message), extra=extra) + super(ResolutionFailure, self).__init__(decode_for_output(message), extra=extra) diff --git a/pipenv/utils.py b/pipenv/utils.py index 7744d9aa17..13c7f635ae 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -30,7 +30,7 @@ import parse from . import environments -from .exceptions import PipenvUsageError +from .exceptions import PipenvUsageError, PipenvCmdError from .pep508checker import lookup from .vendor.urllib3 import util as urllib3_util @@ -119,6 +119,45 @@ def convert_toml_table(section): return parsed +def run_command(cmd, *args, **kwargs): + """ + Take an input command and run it, handling exceptions and error codes and returning + its stdout and stderr. + + :param cmd: The list of command and arguments. + :type cmd: list + :returns: A 2-tuple of the output and error from the command + :rtype: Tuple[str, str] + :raises: exceptions.PipenvCmdError + """ + + from pipenv.vendor import delegator + from ._compat import decode_output + from .cmdparse import Script + if isinstance(cmd, (six.string_types, list, tuple)): + cmd = Script.parse(cmd) + if not isinstance(cmd, Script): + raise TypeError("Command input must be a string, list or tuple") + if "env" not in kwargs: + kwargs["env"] = os.environ.copy() + try: + cmd_string = cmd.cmdify() + except TypeError: + click_echo("Error turning command into string: {0}".format(cmd), err=True) + sys.exit(1) + if environments.is_verbose(): + click_echo("Running command: $ {0}".format(cmd_string, err=True)) + c = delegator.run(cmd_string, *args, **kwargs) + c.block() + if environments.is_verbose(): + click_echo("Command output: {0}".format( + crayons.blue(decode_output(c.out)) + ), err=True) + if not c.ok: + raise PipenvCmdError(cmd_string, c.out, c.err, c.return_code) + return c + + def parse_python_version(output): """Parse a Python version output returned by `python --version`. @@ -782,6 +821,7 @@ def create_spinner(text, nospin=None, spinner_name=None): ) as sp: yield sp + def resolve(cmd, sp): import delegator from .cmdparse import Script @@ -1225,53 +1265,6 @@ def proper_case(package_name): return good_name -def split_section(input_file, section_suffix, test_function): - """ - Split a pipfile or a lockfile section out by section name and test function - - :param dict input_file: A dictionary containing either a pipfile or lockfile - :param str section_suffix: A string of the name of the section - :param func test_function: A test function to test against the value in the key/value pair - - >>> split_section(my_lockfile, 'vcs', is_vcs) - { - 'default': { - "six": { - "hashes": [ - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" - ], - "version": "==1.11.0" - } - }, - 'default-vcs': { - "e1839a8": { - "editable": true, - "path": "." - } - } - } - """ - pipfile_sections = ("packages", "dev-packages") - lockfile_sections = ("default", "develop") - if any(section in input_file for section in pipfile_sections): - sections = pipfile_sections - elif any(section in input_file for section in lockfile_sections): - sections = lockfile_sections - else: - # return the original file if we can't find any pipfile or lockfile sections - return input_file - - for section in sections: - split_dict = {} - entries = input_file.get(section, {}) - for k in list(entries.keys()): - if test_function(entries.get(k)): - split_dict[k] = entries.pop(k) - input_file["-".join([section, section_suffix])] = split_dict - return input_file - - def get_windows_path(*args): """Sanitize a path for windows environments @@ -1854,3 +1847,62 @@ def get_pipenv_dist(pkg="pipenv", pipenv_site=None): pipenv_site = os.path.dirname(pipenv_libdir) pipenv_dist, _ = find_site_path(pkg, site_dir=pipenv_site) return pipenv_dist + + +def find_python(finder, line=None): + """ + Given a `pythonfinder.Finder` instance and an optional line, find a corresponding python + + :param finder: A :class:`pythonfinder.Finder` instance to use for searching + :type finder: :class:pythonfinder.Finder` + :param str line: A version, path, name, or nothing, defaults to None + :return: A path to python + :rtype: str + """ + + if not finder: + from pipenv.vendor.pythonfinder import Finder + finder = Finder(global_search=True) + if not line: + result = next(iter(finder.find_all_python_versions()), None) + elif line and line[0].isnumeric() or re.match(r'[\d\.]+', line): + result = finder.find_python_version(line) + else: + result = finder.find_python_version(name=line) + if not result: + result = finder.which(line) + if not result and not line.startswith("python"): + line = "python{0}".format(line) + result = find_python(finder, line) + if not result: + result = next(iter(finder.find_all_python_versions()), None) + if result: + if not isinstance(result, six.string_types): + return result.path.as_posix() + return result + return + + +def is_python_command(line): + """ + Given an input, checks whether the input is a request for python or notself. + + This can be a version, a python runtime name, or a generic 'python' or 'pythonX.Y' + + :param str line: A potential request to find python + :returns: Whether the line is a python lookup + :rtype: bool + """ + + if not isinstance(line, six.string_types): + raise TypeError("Not a valid command to check: {0!r}".format(line)) + + from pipenv.vendor.pythonfinder.utils import PYTHON_IMPLEMENTATIONS + is_version = re.match(r'[\d\.]+', line) + if line.startswith("python") or is_version or \ + any(line.startswith(v) for v in PYTHON_IMPLEMENTATIONS): + return True + # we are less sure about this but we can guess + if line.startswith("py"): + return True + return False diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a4aca66fbe..18cb94a8e0 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -237,6 +237,29 @@ def test_download_file(self): assert os.path.exists(output) os.remove(output) + @pytest.mark.utils + @pytest.mark.parametrize('line, expected', [ + ("python", True), + ("python3.7", True), + ("python2.7", True), + ("python2", True), + ("python3", True), + ("pypy3", True), + ("anaconda3-5.3.0", True), + ("which", False), + ("vim", False), + ("miniconda", True), + ("micropython", True), + ("ironpython", True), + ("jython3.5", True), + ("2", True), + ("2.7", True), + ("3.7", True), + ("3", True) + ]) + def test_is_python_command(self, line, expected): + assert pipenv.utils.is_python_command(line) == expected + @pytest.mark.utils def test_new_line_end_of_toml_file(this): # toml file that needs clean up From 6c51b7f0ad84f089b688b6c9f2227b77c4cf26a8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 30 Nov 2018 20:11:38 -0500 Subject: [PATCH 02/17] Fix windows lookups Signed-off-by: Dan Ryan --- pipenv/core.py | 2 -- pipenv/utils.py | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 2f88a5304f..cb69f0a965 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -328,8 +328,6 @@ def find_a_system_python(line): * Nothing fits, return None. """ - if os.path.isabs(line): - return line from .vendor.pythonfinder import Finder finder = Finder(system=False, global_search=True) if not line: diff --git a/pipenv/utils.py b/pipenv/utils.py index 13c7f635ae..ad6fd35e2f 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1860,6 +1860,14 @@ def find_python(finder, line=None): :rtype: str """ + if line and not isinstance(line, six.string_types): + raise TypeError( + "Invalid python search type: expected string, received {0!r}".format(line) + ) + if line and os.path.isabs(line): + if os.name == "nt": + line = posixpath.join(*line.split(os.path.sep)) + return line if not finder: from pipenv.vendor.pythonfinder import Finder finder = Finder(global_search=True) From 8da832f44f587a524ab3b847c8556ff60115d62c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Dec 2018 17:37:35 -0500 Subject: [PATCH 03/17] Update exception formatter Signed-off-by: Dan Ryan --- .gitignore | 4 ++++ pipenv/exceptions.py | 2 +- setup.cfg | 9 ++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 44abed16ae..862daf4736 100644 --- a/.gitignore +++ b/.gitignore @@ -145,6 +145,10 @@ venv.bak/ # mypy .mypy_cache/ +# Temporarily generating these with pytype locally for type safety +typeshed/ +pytype.cfg + ### Python Patch ### .venv/ diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 725f382ec9..9007e73e7d 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -33,7 +33,7 @@ def handle_exception(exc_type, exception, traceback, hook=sys.excepthook): line = " {0}".format(line) else: line = " {0}".format(line) - line = "[pipenv.exceptions.{0!s}]: {1}".format( + line = "[{0!s}]: {1}".format( exception.__class__.__name__, line ) click_echo(decode_for_output(line), err=True) diff --git a/setup.cfg b/setup.cfg index 78140fa0c0..896a5a9119 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,7 @@ ignore = # E402: module level import not at top of file # E501: line too long # W503: line break before binary operator - E127,E128,E129,E222,E231,E402,E501,W503 + E402,E501,W503 [isort] atomic=true @@ -29,3 +29,10 @@ known_first_party = pipenv tests ignore_trailing_comma=true + +[mypy] +ignore_missing_imports=true +follow_imports=skip +html_report=mypyhtml +python_version=3.6 +mypy_path=typeshed/pyi:typeshed/imports From fa1c948445e5933f39fac3ae878ce9ecb644b17b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Dec 2018 18:17:00 -0500 Subject: [PATCH 04/17] Minor tweaks Signed-off-by: Dan Ryan --- pipenv/core.py | 19 +++++++++++++++---- pipenv/utils.py | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index cb69f0a965..e7294a2881 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -488,6 +488,7 @@ def abort(): USING_DEFAULT_PYTHON = False # Ensure python is installed before deleting existing virtual env ensure_python(three=three, python=python) + click.echo(crayons.red("Virtualenv already exists!"), err=True) # If VIRTUAL_ENV is set, there is a possibility that we are # going to remove the active virtualenv that the user cares @@ -866,6 +867,7 @@ def convert_three_to_python(three, python): def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): """Creates a virtualenv.""" + click.echo( crayons.normal(fix_utf8("Creating a virtualenv for this project…"), bold=True), err=True ) @@ -875,7 +877,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): ) # Default to using sys.executable, if Python wasn't provided. - if not python: + if python is None: python = sys.executable click.echo( u"{0} {1} {3} {2}".format( @@ -1146,12 +1148,19 @@ def do_init( pypi_mirror=None, ): """Executes the init functionality.""" - from .environments import PIPENV_VIRTUALENV + from .environments import ( + PIPENV_VIRTUALENV, PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_PYTHON, PIPENV_USE_SYSTEM + ) + python = None + if PIPENV_PYTHON is not None: + python = PIPENV_PYTHON + elif PIPENV_DEFAULT_PYTHON_VERSION is not None: + python = PIPENV_DEFAULT_PYTHON_VERSION - if not system: + if not system and not PIPENV_USE_SYSTEM: if not project.virtualenv_exists: try: - do_create_virtualenv(pypi_mirror=pypi_mirror) + do_create_virtualenv(python=python, three=None, pypi_mirror=pypi_mirror) except KeyboardInterrupt: cleanup_virtualenv(bare=False) sys.exit(1) @@ -1522,6 +1531,8 @@ def fallback_which(command, location=None, allow_global=False, system=False): if not isinstance(command, six.string_types): raise TypeError("Provided command must be a string, received {0!r}".format(command)) global_search = system or allow_global + if location is None: + global_search = True finder = Finder(system=False, global_search=global_search, path=location) if is_python_command(command): result = find_python(finder, command) diff --git a/pipenv/utils.py b/pipenv/utils.py index ad6fd35e2f..33fb348015 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1873,7 +1873,7 @@ def find_python(finder, line=None): finder = Finder(global_search=True) if not line: result = next(iter(finder.find_all_python_versions()), None) - elif line and line[0].isnumeric() or re.match(r'[\d\.]+', line): + elif line and line[0].digit() or re.match(r'[\d\.]+', line): result = finder.find_python_version(line) else: result = finder.find_python_version(name=line) From f09e6f5c48af77222e96a09de368f2a8022e25b4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Dec 2018 18:20:22 -0500 Subject: [PATCH 05/17] sort imports for easier merge Signed-off-by: Dan Ryan --- pipenv/core.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index e7294a2881..9b7dfdd770 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -11,7 +11,7 @@ import click import six import urllib3.util as urllib3_util -from pipenv.vendor import vistir +import vistir import click_completion import crayons @@ -2298,13 +2298,6 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror shell_args, ) - # Set an environment variable, so we know we're in the environment. - # Only set PIPENV_ACTIVE after finishing reading virtualenv_location - # otherwise its value will be changed - os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") - - os.environ.pop("PIP_SHIMS_BASE_MODULE", None) - if fancy: shell.fork(*fork_args) return From cfd1e5992e1be68d2488e0aac6afc1385c0d1361 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Dec 2018 18:45:46 -0500 Subject: [PATCH 06/17] Make sure python path is a string Signed-off-by: Dan Ryan --- pipenv/core.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 9b7dfdd770..075fcb6935 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -466,6 +466,8 @@ def abort(): ensure_environment() # Ensure Python is available. python = ensure_python(three=three, python=python) + if python is not None and not isinstance(python, six.string_types): + python = python.path.as_posix() # Create the virtualenv. # Abort if --system (or running in a virtualenv). if PIPENV_USE_SYSTEM: @@ -487,7 +489,9 @@ def abort(): elif (python) or (three is not None) or (site_packages is not False): USING_DEFAULT_PYTHON = False # Ensure python is installed before deleting existing virtual env - ensure_python(three=three, python=python) + python = ensure_python(three=three, python=python) + if python is not None and not isinstance(python, six.string_types): + python = python.path.as_posix() click.echo(crayons.red("Virtualenv already exists!"), err=True) # If VIRTUAL_ENV is set, there is a possibility that we are From 77bac23f93f4750342b43c72a87a98d1ea91d313 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 10 Mar 2019 17:32:47 -0400 Subject: [PATCH 07/17] re-add setting to make environment active when running shell Signed-off-by: Dan Ryan --- pipenv/core.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 075fcb6935..d5d6ec52af 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1441,7 +1441,7 @@ def pip_install( ignore_hashes = True else: ignore_hashes = True if not requirement.hashes else False - install_reqs = requirement.as_line(as_list=True) + install_reqs = requirement.as_line(as_list=True, include_hashes=not ignore_hashes) if not requirement.markers: install_reqs = [escape_cmd(r) for r in install_reqs] elif len(install_reqs) > 1: @@ -2301,6 +2301,12 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror project.project_directory, shell_args, ) + # Only set PIPENV_ACTIVE after finishing reading virtualenv_location + # Set an environment variable, so we know we're in the environment. + # otherwise its value will be changed + os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") + + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) if fancy: shell.fork(*fork_args) From 089805712a078fad3a2824926613e6fd798a805c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 10 Mar 2019 20:25:21 -0400 Subject: [PATCH 08/17] Update safety check command runner to handle exceptions Signed-off-by: Dan Ryan --- pipenv/core.py | 18 +++++++++++++----- pipenv/utils.py | 11 ++++++----- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index d5d6ec52af..298aab239e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2569,7 +2569,9 @@ def do_check( decode_for_output("Checking installed package safety…"), bold=True) ) if ignore: - ignored = "--ignore {0}".format(" --ignore ".join(ignore)) + if not isinstance(ignore, (tuple, list)): + ignore = [ignore] + ignored = [["--ignore", cve] for cve in ignore] click.echo( crayons.normal( "Notice: Ignoring CVE(s) {0}".format(crayons.yellow(", ".join(ignore))) @@ -2579,12 +2581,20 @@ def do_check( else: ignored = "" key = "--key={0}".format(PIPENV_PYUP_API_KEY) - cmd = _cmd + [safety_path, "check", "--json", key, ignored] - c = run_command(cmd) + cmd = _cmd + [safety_path, "check", "--json", key] + if ignored: + for cve in ignored: + cmd += cve + c = run_command(cmd, catch_exceptions=False) try: results = simplejson.loads(c.out) except (ValueError, JSONDecodeError): raise exceptions.JSONParseError(c.out, c.err) + except Exception: + raise exceptions.PipenvCmdError(c.cmd, c.out, c.err, c.return_code) + if c.ok: + click.echo(crayons.green("All good!")) + sys.exit(0) for (package, resolved, installed, description, vuln) in results: click.echo( "{0}: {1} {2} resolved ({3} installed)!".format( @@ -2596,8 +2606,6 @@ def do_check( ) click.echo("{0}".format(description)) click.echo() - if not results: - click.echo(crayons.green("All good!")) else: sys.exit(1) diff --git a/pipenv/utils.py b/pipenv/utils.py index 33fb348015..96d2173a49 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -119,13 +119,14 @@ def convert_toml_table(section): return parsed -def run_command(cmd, *args, **kwargs): +def run_command(cmd, *args, catch_exceptions=True, **kwargs): """ Take an input command and run it, handling exceptions and error codes and returning its stdout and stderr. :param cmd: The list of command and arguments. :type cmd: list + :param bool catch_exceptions: Whether to catch and raise exceptions on failure :returns: A 2-tuple of the output and error from the command :rtype: Tuple[str, str] :raises: exceptions.PipenvCmdError @@ -153,7 +154,7 @@ def run_command(cmd, *args, **kwargs): click_echo("Command output: {0}".format( crayons.blue(decode_output(c.out)) ), err=True) - if not c.ok: + if not c.ok and catch_exceptions: raise PipenvCmdError(cmd_string, c.out, c.err, c.return_code) return c @@ -1873,7 +1874,7 @@ def find_python(finder, line=None): finder = Finder(global_search=True) if not line: result = next(iter(finder.find_all_python_versions()), None) - elif line and line[0].digit() or re.match(r'[\d\.]+', line): + elif line and line[0].isdigit() or re.match(r'[\d\.]+', line): result = finder.find_python_version(line) else: result = finder.find_python_version(name=line) @@ -1907,8 +1908,8 @@ def is_python_command(line): from pipenv.vendor.pythonfinder.utils import PYTHON_IMPLEMENTATIONS is_version = re.match(r'[\d\.]+', line) - if line.startswith("python") or is_version or \ - any(line.startswith(v) for v in PYTHON_IMPLEMENTATIONS): + if (line.startswith("python") or is_version or + any(line.startswith(v) for v in PYTHON_IMPLEMENTATIONS)): return True # we are less sure about this but we can guess if line.startswith("py"): From e08ce4de05b7111de16764da49ba575e50c4068a Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 11 Mar 2019 00:58:02 -0400 Subject: [PATCH 09/17] fix syntax error on python 2 Signed-off-by: Dan Ryan --- pipenv/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 96d2173a49..d29e82836f 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -119,22 +119,22 @@ def convert_toml_table(section): return parsed -def run_command(cmd, *args, catch_exceptions=True, **kwargs): +def run_command(cmd, *args, **kwargs): """ Take an input command and run it, handling exceptions and error codes and returning its stdout and stderr. :param cmd: The list of command and arguments. :type cmd: list - :param bool catch_exceptions: Whether to catch and raise exceptions on failure :returns: A 2-tuple of the output and error from the command :rtype: Tuple[str, str] :raises: exceptions.PipenvCmdError """ from pipenv.vendor import delegator - from ._compat import decode_output + from ._compat import decode_for_output from .cmdparse import Script + catch_exceptions = kwargs.pop("catch_exceptions", True) if isinstance(cmd, (six.string_types, list, tuple)): cmd = Script.parse(cmd) if not isinstance(cmd, Script): @@ -152,7 +152,7 @@ def run_command(cmd, *args, catch_exceptions=True, **kwargs): c.block() if environments.is_verbose(): click_echo("Command output: {0}".format( - crayons.blue(decode_output(c.out)) + crayons.blue(decode_for_output(c.out)) ), err=True) if not c.ok and catch_exceptions: raise PipenvCmdError(cmd_string, c.out, c.err, c.return_code) @@ -438,7 +438,7 @@ def get_deps_from_req(cls, req): new_constraints = {} _, new_entry = req.pipfile_entry new_lock = { - pep_423_name(new_req.normalized_name): new_entry + pep423_name(new_req.normalized_name): new_entry } else: new_constraints, new_lock = cls.get_deps_from_req(new_req) From 8fdc589d0531272a52ee5e5c06d0821f841749ab Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 11 Mar 2019 01:51:05 -0400 Subject: [PATCH 10/17] Fix test skip Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 8 ++++++++ tests/integration/test_install_markers.py | 8 ++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e400fc0663..b7c4784b4a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -85,6 +85,14 @@ def pytest_runtest_setup(item): sys.version_info[:2] <= (2, 7) and os.name == "nt" ): pytest.skip('must use python > 2.7 on windows') + if item.get_marker('py3_only') is not None and ( + sys.version_info[:2] < (3, 0) + ): + pytest.mark.skip('test only runs on python 3') + if item.get_marker('lte_py36') is not None and ( + sys.version_info[:2] >= (3, 7) + ): + pytest.mark.skip('test only runs on python < 3.7') @pytest.fixture diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index bee1d994d3..58288dc1fa 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -10,10 +10,6 @@ from pipenv.utils import temp_environ -py3_only = pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3") -skip_py37 = pytest.mark.skipif(sys.version_info >= (3, 7), reason="Skip for python 3.7") - - @pytest.mark.markers @flaky def test_package_environment_markers(PipenvInstance, pypi): @@ -130,9 +126,9 @@ def test_global_overrides_environment_markers(PipenvInstance, pypi): @pytest.mark.lock @pytest.mark.complex +@pytest.mark.py3_only +@pytest.mark.lte_py36 @flaky -@py3_only -@skip_py37 def test_resolver_unique_markers(PipenvInstance, pypi): """vcrpy has a dependency on `yarl` which comes with a marker of 'python version in "3.4, 3.5, 3.6" - this marker duplicates itself: From 73129d8401bdd1d5399359a02d9213805026804d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 11 Mar 2019 19:42:06 -0400 Subject: [PATCH 11/17] Fix internet connectivity test Signed-off-by: Dan Ryan --- pytest.ini | 18 +++++++++++++++++- tests/integration/conftest.py | 26 +++++++++++++++++--------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/pytest.ini b/pytest.ini index ff9847d3b1..61b492e331 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,23 @@ addopts = -ra -n auto testpaths = tests ; Add vendor and patched in addition to the default list of ignored dirs -norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched news tasks docs tests/test_artifacts +; Additionally, ignore tasks, news, test subdirectories and peeps directory +norecursedirs = + .* build + dist + CVS + _darcs + {arch} + *.egg + vendor + patched + news + tasks + docs + tests/test_artifacts + tests/pytest-pypi + tests/pypi + peeps filterwarnings = ignore::DeprecationWarning ignore::PendingDeprecationWarning diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b7c4784b4a..ac99418d01 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -23,16 +23,24 @@ HAS_WARNED_GITHUB = False +def try_internet(url="http://httpbin.org/ip", timeout=1.5): + resp = requests.get(url, timeout=timeout) + resp.raise_for_status() + + def check_internet(): - try: - # Kenneth represents the Internet LGTM. - resp = requests.get('http://httpbin.org/ip', timeout=1.0) - resp.raise_for_status() - except Exception: - warnings.warn('Cannot connect to HTTPBin...', RuntimeWarning) - warnings.warn('Will skip tests requiring Internet', RuntimeWarning) - return False - return True + has_internet = False + for url in ("http://httpbin.org/ip", "http://clients3.google.com/generate_204"): + try: + try_internet(url) + except Exception: + warnings.warn( + "Failed connecting to internet: {0}".format(url), RuntimeWarning + ) + else: + has_internet = True + break + return has_internet def check_github_ssh(): From 466fcc0a2a9fdee6cabbb22cfb99ed896689ef5f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 11 Mar 2019 20:19:27 -0400 Subject: [PATCH 12/17] Fix test skip Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ac99418d01..446f5b4d27 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -94,13 +94,13 @@ def pytest_runtest_setup(item): ): pytest.skip('must use python > 2.7 on windows') if item.get_marker('py3_only') is not None and ( - sys.version_info[:2] < (3, 0) + sys.version_info < (3, 0) ): - pytest.mark.skip('test only runs on python 3') + pytest.skip('test only runs on python 3') if item.get_marker('lte_py36') is not None and ( - sys.version_info[:2] >= (3, 7) + sys.version_info >= (3, 7) ): - pytest.mark.skip('test only runs on python < 3.7') + pytest.skip('test only runs on python < 3.7') @pytest.fixture From dcd5369117f44b30d6afee0d10322b79db540612 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 12 Mar 2019 01:38:47 -0400 Subject: [PATCH 13/17] Fix vistir's encoding of terminal output for python 2 Signed-off-by: Dan Ryan --- pipenv/__init__.py | 13 ++++++++ pipenv/core.py | 5 ++-- pipenv/vendor/vistir/misc.py | 1 + pipenv/vendor/vistir/spin.py | 57 ++++++++++++++++++------------------ 4 files changed, 46 insertions(+), 30 deletions(-) diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 7a12158aee..769ede2045 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -26,6 +26,8 @@ warnings.filterwarnings("ignore", category=ResourceWarning) warnings.filterwarnings("ignore", category=UserWarning) + + if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): if sys.stdout.isatty() and sys.stderr.isatty(): import io @@ -46,6 +48,17 @@ except Exception: pass +if sys.version_info >= (3, 0): + stdout = sys.stdout.buffer + stderr = sys.stderr.buffer +else: + stdout = sys.stdout + stderr = sys.stderr + +from .vendor.vistir.misc import get_wrapped_stream +sys.stderr = get_wrapped_stream(stderr) +sys.stdout = get_wrapped_stream(stdout) + from .cli import cli from . import resolver diff --git a/pipenv/core.py b/pipenv/core.py index 298aab239e..05dba22f53 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -923,12 +923,13 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): ) click.echo(crayons.blue("{0}".format(c.out)), err=True) if c.returncode != 0: - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Failed creating virtual environment")) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format(u"Failed creating virtual environment")) raise exceptions.VirtualenvCreationException( extra=[crayons.blue("{0}".format(c.err)),] ) else: - sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Successfully created virtual environment!")) + + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format(u"Successfully created virtual environment!")) # Associate project directory with the environment. # This mimics Pew's "setproject". diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 42067b2d2f..fe88dc1fd1 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -590,6 +590,7 @@ def decode_for_output(output, target_stream=None, translation_map=None): try: output = _encode(output, encoding=encoding, translation_map=translation_map) except (UnicodeDecodeError, UnicodeEncodeError): + output = to_native_string(output) output = _encode( output, encoding=encoding, errors="replace", translation_map=translation_map ) diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 40dfff962a..d7b6dfa6e8 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -14,7 +14,7 @@ from .compat import to_native_string from .cursor import hide_cursor, show_cursor -from .misc import decode_for_output +from .misc import decode_for_output, to_text from .termcolors import COLOR_MAP, COLORS, DISABLE_COLORS, colored try: @@ -131,9 +131,9 @@ def hide_and_write(self, text, target=None): target = self.stdout if text is None or isinstance(text, six.string_types) and text == "None": pass - target.write(decode_output("\r", target_stream=target)) + target.write(decode_output(u"\r", target_stream=target)) self._hide_cursor(target=target) - target.write(decode_output("{0}\n".format(text), target_stream=target)) + target.write(decode_output(u"{0}\n".format(text), target_stream=target)) target.write(CLEAR_LINE) self._show_cursor(target=target) @@ -146,9 +146,8 @@ def write(self, text=None): stdout = self.stdout else: stdout = sys.stdout - text = decode_output(text, target_stream=stdout) - stdout.write(decode_output("\r", target_stream=stdout)) - line = decode_output("{0}\n".format(text), target_stream=stdout) + stdout.write(decode_output(u"\r", target_stream=stdout)) + line = decode_output(u"{0}\n".format(text), target_stream=stdout) stdout.write(line) stdout.write(CLEAR_LINE) @@ -162,9 +161,8 @@ def write_err(self, text=None): print(text) return stderr = sys.stderr - text = decode_output(text, target_stream=stderr) - stderr.write(decode_output("\r", target_stream=stderr)) - line = decode_output("{0}\n".format(text), target_stream=stderr) + stderr.write(decode_output(u"\r", target_stream=stderr)) + line = decode_output(u"{0}\n".format(text), target_stream=stderr) stderr.write(line) stderr.write(CLEAR_LINE) @@ -224,32 +222,32 @@ def __init__(self, *args, **kwargs): if DISABLE_COLORS: colorama.deinit() - def ok(self, text="OK", err=False): + def ok(self, text=u"OK", err=False): """Set Ok (success) finalizer to a spinner.""" # Do not display spin text for ok state self._text = None - _text = text if text else "OK" + _text = to_text(text) if text else u"OK" err = err or not self.write_to_stdout self._freeze(_text, err=err) - def fail(self, text="FAIL", err=False): + def fail(self, text=u"FAIL", err=False): """Set fail finalizer to a spinner.""" # Do not display spin text for fail state self._text = None - _text = text if text else "FAIL" + _text = text if text else u"FAIL" err = err or not self.write_to_stdout self._freeze(_text, err=err) def hide_and_write(self, text, target=None): if not target: target = self.stdout - if text is None or isinstance(text, six.string_types) and text == "None": + if text is None or isinstance(text, six.string_types) and text == u"None": pass - target.write(decode_output("\r")) + target.write(decode_output(u"\r")) self._hide_cursor(target=target) - target.write(decode_output("{0}\n".format(text))) + target.write(decode_output(u"{0}\n".format(text))) target.write(CLEAR_LINE) self._show_cursor(target=target) @@ -259,24 +257,24 @@ def write(self, text): stdout = self.stdout if self.stdout.closed: stdout = sys.stdout - stdout.write(decode_output("\r", target_stream=stdout)) + stdout.write(decode_output(u"\r", target_stream=stdout)) stdout.write(decode_output(CLEAR_LINE, target_stream=stdout)) if text is None: text = "" - text = decode_output("{0}\n".format(text), target_stream=stdout) + text = decode_output(u"{0}\n".format(text), target_stream=stdout) stdout.write(text) - self.out_buff.write(decode_output(text, target_stream=self.out_buff)) + self.out_buff.write(text) def write_err(self, text): """Write error text in the terminal without breaking the spinner.""" stderr = self.stderr if self.stderr.closed: stderr = sys.stderr - stderr.write(decode_output("\r", target_stream=stderr)) + stderr.write(decode_output(u"\r", target_stream=stderr)) stderr.write(decode_output(CLEAR_LINE, target_stream=stderr)) if text is None: text = "" - text = decode_output("{0}\n".format(text), target_stream=stderr) + text = decode_output(u"{0}\n".format(text), target_stream=stderr) self.stderr.write(text) self.out_buff.write(decode_output(text, target_stream=self.out_buff)) @@ -322,8 +320,9 @@ def _freeze(self, final_text, err=False): target = self.stderr if err else self.stdout if target.closed: target = sys.stderr if err else sys.stdout - text = decode_output(final_text, target_stream=target) - self._last_frame = self._compose_out(text, mode="last") + text = to_text(final_text) + last_frame = self._compose_out(text, mode="last") + self._last_frame = decode_output(last_frame, target_stream=target) # Should be stopped here, otherwise prints after # self._freeze call will mess up the spinner @@ -339,19 +338,20 @@ def _compose_color_func(self): def _compose_out(self, frame, mode=None): # Ensure Unicode input - frame = decode_output(frame) + frame = to_text(frame) if self._text is None: - self._text = "" - text = decode_output(self._text) + self._text = u"" + text = to_text(self._text) if self._color_func is not None: frame = self._color_func(frame) if self._side == "right": frame, text = text, frame # Mode + frame = to_text(frame) if not mode: - out = decode_output("\r{0} {1}".format(frame, text)) + out = u"\r{0} {1}".format(frame, text) else: - out = decode_output("{0} {1}\n".format(frame, text)) + out = u"{0} {1}\n".format(frame, text) return out def _spin(self): @@ -367,6 +367,7 @@ def _spin(self): # Compose output spin_phase = next(self._cycle) out = self._compose_out(spin_phase) + out = decode_output(out, target) # Write target.write(out) From a17760e223e9c2e4d21e2afe5d6ce2a37b196bdf Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 12 Mar 2019 02:00:35 -0400 Subject: [PATCH 14/17] Update dummy spinner Signed-off-by: Dan Ryan --- pipenv/vendor/vistir/spin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index d7b6dfa6e8..877ece82f7 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -147,6 +147,7 @@ def write(self, text=None): else: stdout = sys.stdout stdout.write(decode_output(u"\r", target_stream=stdout)) + text = to_text(text) line = decode_output(u"{0}\n".format(text), target_stream=stdout) stdout.write(line) stdout.write(CLEAR_LINE) @@ -154,6 +155,7 @@ def write(self, text=None): def write_err(self, text=None): if text is None or isinstance(text, six.string_types) and text == "None": pass + text = to_text(text) if not self.stderr.closed: stderr = self.stderr else: From d361c86c7df3ad44a5e9a27cf8c17e6daa9517b3 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 12 Mar 2019 22:18:35 -0400 Subject: [PATCH 15/17] Update azure pipeline scripts Signed-off-by: Dan Ryan Update azure builds Signed-off-by: Dan Ryan Update azure pipeline windows code Signed-off-by: Dan Ryan Update python executable search for windows Signed-off-by: Dan Ryan Fix variable reference Signed-off-by: Dan Ryan Set virtual env variable on windows Signed-off-by: Dan Ryan globally install pipenv on windows Signed-off-by: Dan Ryan --- .azure-pipelines/jobs/run-tests-windows.yml | 4 ++++ .azure-pipelines/jobs/run-tests.yml | 19 +++++-------------- .../steps/create-virtualenv-linux.yml | 14 ++++++++++++++ .azure-pipelines/steps/create-virtualenv.yml | 12 ++++++++++-- .../steps/install-dependencies.yml | 2 +- 5 files changed, 34 insertions(+), 17 deletions(-) create mode 100644 .azure-pipelines/steps/create-virtualenv-linux.yml diff --git a/.azure-pipelines/jobs/run-tests-windows.yml b/.azure-pipelines/jobs/run-tests-windows.yml index 6b6f86fa3b..161247e318 100644 --- a/.azure-pipelines/jobs/run-tests-windows.yml +++ b/.azure-pipelines/jobs/run-tests-windows.yml @@ -4,6 +4,10 @@ steps: inputs: versionSpec: '$(python.version)' architecture: '$(python.architecture)' + addToPath: true + +- script: | + echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]$(python.version)' - template: ../steps/install-dependencies.yml diff --git a/.azure-pipelines/jobs/run-tests.yml b/.azure-pipelines/jobs/run-tests.yml index 2602cba67e..c83e62a05b 100644 --- a/.azure-pipelines/jobs/run-tests.yml +++ b/.azure-pipelines/jobs/run-tests.yml @@ -4,23 +4,14 @@ steps: inputs: versionSpec: '$(python.version)' architecture: '$(python.architecture)' + addToPath: true + +- script: | + echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]$(python.version)' - template: ../steps/install-dependencies.yml -- bash: | - mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" - mkdir -p "$WORKON_HOME" - pip install certifi - export GIT_SSL_CAINFO="$(python -m certifi)" - export LANG="C.UTF-8" - export PIP_PROCESS_DEPENDENCY_LINKS="1" - echo "Path $PATH" - echo "Installing Pipenv…" - pip install -e "$(pwd)[test]" --upgrade - pipenv install --deploy --dev - pipenv run pip install -e "$(pwd)[test]" --upgrade - echo pipenv --venv && echo pipenv --py && echo pipenv run python --version - displayName: Make Virtualenv +- template: ../steps/create-virtualenv-linux.yml - script: | # Fix Git SSL errors diff --git a/.azure-pipelines/steps/create-virtualenv-linux.yml b/.azure-pipelines/steps/create-virtualenv-linux.yml new file mode 100644 index 0000000000..e53893763e --- /dev/null +++ b/.azure-pipelines/steps/create-virtualenv-linux.yml @@ -0,0 +1,14 @@ +steps: +- bash: | + mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" + mkdir -p "$WORKON_HOME" + pip install certifi + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" + export PIP_PROCESS_DEPENDENCY_LINKS="1" + echo "Path $PATH" + echo "Installing Pipenv…" + pipenv install --deploy --dev + pipenv run pip install -e "$(pwd)[test]" --upgrade + echo pipenv --venv && echo pipenv --py && echo pipenv run python --version + displayName: Make Virtualenv diff --git a/.azure-pipelines/steps/create-virtualenv.yml b/.azure-pipelines/steps/create-virtualenv.yml index 60ade40b3e..14f0a0e81f 100644 --- a/.azure-pipelines/steps/create-virtualenv.yml +++ b/.azure-pipelines/steps/create-virtualenv.yml @@ -1,6 +1,14 @@ steps: +- powershell: | + Write-Host "##vso[task.setvariable variable=PY_EXE]"(py -"$PIPENV_DEFAULT_PYTHON_VERSION" -c 'import sys; print(sys.executable)') + - script: | - virtualenv D:\.venv - D:\.venv\Scripts\pip.exe install -e .[test] && D:\.venv\Scripts\pipenv install --dev && D:\.venv\Scripts\pipenv run pip install -e .[test] + echo "Python exe: "$(PY_EXE) + virtualenv --python=$(PY_EXE) D:\.venv + echo "##vso[task.setvariable variable=VIRTUAL_ENV]"$(PY_EXE) + python -m pip install -e .[test] --upgrade + D:\.venv\Scripts\pip.exe install -e .[test] --upgrade + D:\.venv\Scripts\pipenv install --dev + D:\.venv\Scripts\pipenv run pip install -e .[test] echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version displayName: Make Virtualenv diff --git a/.azure-pipelines/steps/install-dependencies.yml b/.azure-pipelines/steps/install-dependencies.yml index 16b3d6b8b3..fd0da8415d 100644 --- a/.azure-pipelines/steps/install-dependencies.yml +++ b/.azure-pipelines/steps/install-dependencies.yml @@ -1,3 +1,3 @@ steps: -- script: 'python -m pip install --upgrade pip && python -m pip install -e .[test]' +- script: 'python -m pip install --upgrade pip setuptools wheel && python -m pip install -e .[test] --upgrade' displayName: Upgrade Pip & Install Pipenv From a23e9246d63189537f95e9cb5bfcf3ee4bc4f132 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 15 Mar 2019 12:38:34 -0400 Subject: [PATCH 16/17] Update vistir Signed-off-by: Dan Ryan --- pipenv/vendor/vistir/path.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index d551aac781..9eb113798f 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -19,6 +19,7 @@ Path, ResourceWarning, TemporaryDirectory, + FileNotFoundError, _fs_encoding, _NamedTemporaryFile, finalize, From 116d0857a20d38a866a840f2d669e836a2aa3858 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 13 Mar 2019 20:06:23 -0400 Subject: [PATCH 17/17] Fix variable references Signed-off-by: Dan Ryan Fix variable references Signed-off-by: Dan Ryan Fix variable references Signed-off-by: Dan Ryan Switch to powershell for windows virtualenv creation Signed-off-by: Dan Ryan use python version as path Signed-off-by: Dan Ryan use python version as path Signed-off-by: Dan Ryan swap variable name for python exe Signed-off-by: Dan Ryan add fallback for python exe Signed-off-by: Dan Ryan fix python variable setting Signed-off-by: Dan Ryan fix python variable setting Signed-off-by: Dan Ryan Use variable susbstitution for python executable location Signed-off-by: Dan Ryan Use activate script properly Signed-off-by: Dan Ryan Fix floating quote in python version Signed-off-by: Dan Ryan Don't block on safety call in python 2 as it overwrites output for some reason Signed-off-by: Dan Ryan Don't block on pipenv graph either Signed-off-by: Dan Ryan Check command return code instead of calling `block` Signed-off-by: Dan Ryan Don't load json after its already loaded Signed-off-by: Dan Ryan Wait on return code before checking contents Signed-off-by: Dan Ryan --- .azure-pipelines/jobs/run-tests-windows.yml | 6 ++- .azure-pipelines/steps/create-virtualenv.yml | 38 ++++++++++++------ pipenv/__init__.py | 16 ++------ pipenv/core.py | 41 ++++++++++++++------ pipenv/utils.py | 5 ++- tests/integration/test_install_basic.py | 1 + 6 files changed, 67 insertions(+), 40 deletions(-) diff --git a/.azure-pipelines/jobs/run-tests-windows.yml b/.azure-pipelines/jobs/run-tests-windows.yml index 161247e318..700732c52b 100644 --- a/.azure-pipelines/jobs/run-tests-windows.yml +++ b/.azure-pipelines/jobs/run-tests-windows.yml @@ -6,8 +6,10 @@ steps: architecture: '$(python.architecture)' addToPath: true -- script: | - echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]$(python.version)' +- powershell: | + Write-Host "##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]$env:PYTHON_VERSION" + env: + PYTHON_VERSION: $(python.version) - template: ../steps/install-dependencies.yml diff --git a/.azure-pipelines/steps/create-virtualenv.yml b/.azure-pipelines/steps/create-virtualenv.yml index 14f0a0e81f..3151d15ecf 100644 --- a/.azure-pipelines/steps/create-virtualenv.yml +++ b/.azure-pipelines/steps/create-virtualenv.yml @@ -1,14 +1,30 @@ steps: -- powershell: | - Write-Host "##vso[task.setvariable variable=PY_EXE]"(py -"$PIPENV_DEFAULT_PYTHON_VERSION" -c 'import sys; print(sys.executable)') -- script: | - echo "Python exe: "$(PY_EXE) - virtualenv --python=$(PY_EXE) D:\.venv - echo "##vso[task.setvariable variable=VIRTUAL_ENV]"$(PY_EXE) - python -m pip install -e .[test] --upgrade - D:\.venv\Scripts\pip.exe install -e .[test] --upgrade - D:\.venv\Scripts\pipenv install --dev - D:\.venv\Scripts\pipenv run pip install -e .[test] - echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version +- powershell: | + $env:PY_EXE=$(python -c "import sys; print(sys.executable)") + if (!$env:PY_EXE) { + $env:PY_EXE="python" + } + Write-Host "##vso[task.setvariable variable=PY_EXE]"$env:PY_EXE + Write-Host "Found Python: $env:PY_EXE" + Invoke-Expression "$env:PY_EXE -m virtualenv D:\.venv" + Write-Host "##vso[task.setvariable variable=VIRTUAL_ENV]D:\.venv" + Invoke-Expression "D:\.venv\Scripts\activate.ps1" + $env:VIRTUAL_ENV="D:\.venv" + Write-Host "Installing local package..." + Invoke-Expression "$env:PY_EXE -m pip install -e .[test] --upgrade" + Write-Host "upgrading local package in virtual env" + $venv_scripts = Join-Path -path D:\.venv -childpath Scripts + $venv_py = Join-Path -path $venv_scripts -childpath python.exe + Invoke-Expression "$venv_py -m pip install -e .[test] --upgrade" + Write-Host "Installing pipenv development packages" + Invoke-Expression "$venv_py -m pipenv install --dev" + Write-Host "Installing local package in pipenv environment" + Invoke-Expression "$venv_py -m pipenv run pip install -e .[test]" + Write-Host "Printing metadata" + Write-Host $(Invoke-Expression "$venv_py -m pipenv --venv") + Write-Host $(Invoke-Expression "$venv_py -m pipenv --py") + Write-Host $(Invoke-Expression "$venv_py -m pipenv run python --version") displayName: Make Virtualenv + env: + PIPENV_DEFAULT_PYTHON_VERSION: $(PIPENV_DEFAULT_PYTHON_VERSION) diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 769ede2045..51aafed7ba 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -27,18 +27,6 @@ warnings.filterwarnings("ignore", category=UserWarning) - -if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): - if sys.stdout.isatty() and sys.stderr.isatty(): - import io - import atexit - stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') - atexit.register(stdout_wrapper.close) - stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') - atexit.register(stderr_wrapper.close) - sys.stdout = stdout_wrapper - sys.stderr = stderr_wrapper - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") # Hack to make things work better. @@ -48,6 +36,7 @@ except Exception: pass +from .vendor.vistir.misc import get_wrapped_stream if sys.version_info >= (3, 0): stdout = sys.stdout.buffer stderr = sys.stderr.buffer @@ -55,10 +44,11 @@ stdout = sys.stdout stderr = sys.stderr -from .vendor.vistir.misc import get_wrapped_stream + sys.stderr = get_wrapped_stream(stderr) sys.stdout = get_wrapped_stream(stdout) + from .cli import cli from . import resolver diff --git a/pipenv/core.py b/pipenv/core.py index 05dba22f53..8e2059a608 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2490,6 +2490,7 @@ def do_check( args=None, pypi_mirror=None, ): + from .environments import is_verbose from pipenv.vendor.vistir.compat import JSONDecodeError if not system: # Ensure that virtualenv is available. @@ -2530,18 +2531,25 @@ def do_check( python = which("python") else: python = system_which("python") - _cmd = [python,] + _cmd = [vistir.compat.Path(python).as_posix()] # Run the PEP 508 checker in the virtualenv. - cmd = _cmd + [pep508checker_path] + cmd = _cmd + [vistir.compat.Path(pep508checker_path).as_posix()] c = run_command(cmd) - try: - results = simplejson.loads(c.out.strip()) - except JSONDecodeError: - click.echo("{0}\n{1}".format( - crayons.white(decode_for_output("Failed parsing pep508 results: "), bold=True), - c.out.strip() + if is_verbose(): + click.echo("{0}{1}".format( + "Running command: ", + crayons.white("$ {0}".format(decode_for_output(" ".join(cmd))), bold=True) )) - sys.exit(1) + if c.return_code is not None: + try: + results = simplejson.loads(c.out.strip()) + except JSONDecodeError: + click.echo("{0}\n{1}\n{2}".format( + crayons.white(decode_for_output("Failed parsing pep508 results: "), bold=True), + c.out.strip(), + c.err.strip() + )) + sys.exit(1) # Load the pipfile. p = pipfile.Pipfile.load(project.pipfile_location) failed = False @@ -2629,6 +2637,9 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): sys.exit(1) except RuntimeError: pass + else: + python_path = vistir.compat.Path(python_path).as_posix() + pipdeptree_path = vistir.compat.Path(pipdeptree_path).as_posix() if reverse and json: click.echo( @@ -2685,9 +2696,14 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): if not bare: if json: data = [] - for d in simplejson.loads(c.out): - if d["package"]["key"] not in BAD_PACKAGES: - data.append(d) + try: + parsed = simplejson.loads(c.out.strip()) + except JSONDecodeError: + raise exceptions.JSONParseError(c.out, c.err) + else: + for d in parsed: + if d["package"]["key"] not in BAD_PACKAGES: + data.append(d) click.echo(simplejson.dumps(data, indent=4)) sys.exit(0) elif json_tree: @@ -2714,6 +2730,7 @@ def traverse(obj): else: for line in c.out.strip().split("\n"): # Ignore bad packages as top level. + # TODO: This should probably be a "==" in + line.partition if line.split("==")[0] in BAD_PACKAGES and not reverse: continue diff --git a/pipenv/utils.py b/pipenv/utils.py index d29e82836f..7b43c1f004 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -141,6 +141,7 @@ def run_command(cmd, *args, **kwargs): raise TypeError("Command input must be a string, list or tuple") if "env" not in kwargs: kwargs["env"] = os.environ.copy() + kwargs["env"]["PYTHONIOENCODING"] = "UTF-8" try: cmd_string = cmd.cmdify() except TypeError: @@ -149,13 +150,13 @@ def run_command(cmd, *args, **kwargs): if environments.is_verbose(): click_echo("Running command: $ {0}".format(cmd_string, err=True)) c = delegator.run(cmd_string, *args, **kwargs) - c.block() + return_code = c.return_code if environments.is_verbose(): click_echo("Command output: {0}".format( crayons.blue(decode_for_output(c.out)) ), err=True) if not c.ok and catch_exceptions: - raise PipenvCmdError(cmd_string, c.out, c.err, c.return_code) + raise PipenvCmdError(cmd_string, c.out, c.err, return_code) return c diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index 3521ee5bce..1af9b6af30 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -315,6 +315,7 @@ def test_skip_requirements_when_pipfile(PipenvInstance, pypi): """.strip() f.write(contents) c = p.pipenv("install") + assert c.ok assert "tablib" in p.pipfile["packages"] assert "tablib" in p.lockfile["default"] assert "six" in p.pipfile["packages"]