diff --git a/news/2564.bugfix b/news/2564.bugfix new file mode 100644 index 0000000000..31fc0b1a84 --- /dev/null +++ b/news/2564.bugfix @@ -0,0 +1,2 @@ +Fixed a bug related to parsing vcs requirements with both extras and subdirectory fragments. +Corrected an issue in the ``requirementslib`` parser which led to some markers being discarded rather than evaluated. diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 776d2a15bc..063458dc7e 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = "1.0.9" +__version__ = "1.0.10" from .exceptions import RequirementError diff --git a/pipenv/vendor/requirementslib/models/baserequirement.py b/pipenv/vendor/requirementslib/models/baserequirement.py index 0cac987af4..b97dee40a1 100644 --- a/pipenv/vendor/requirementslib/models/baserequirement.py +++ b/pipenv/vendor/requirementslib/models/baserequirement.py @@ -28,3 +28,10 @@ def pipfile_part(self): @classmethod def attr_fields(cls): return [field.name for field in attr.fields(cls)] + + @property + def extras_as_pip(self): + if self.extras: + return "[{0}]".format(",".join(self.extras)) + + return "" diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/vendor/requirementslib/models/markers.py index c3df449948..70adc21f8c 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/vendor/requirementslib/models/markers.py @@ -66,21 +66,14 @@ def make_marker(cls, marker_string): raise RequirementError( "Invalid requirement: Invalid marker %r" % marker_string ) - marker_dict = {} - for m in marker._markers: - if isinstance(m, six.string_types): - continue - var, op, val = m - if var.value in cls.attr_fields(): - marker_dict[var.value] = '{0} "{1}"'.format(op, val) - return marker_dict + return marker @classmethod def from_line(cls, line): if ";" in line: line = line.rsplit(";", 1)[1].strip() - marker_dict = cls.make_marker(line) - return cls(**marker_dict) + marker = cls.make_marker(line) + return marker @classmethod def from_pipfile(cls, name, pipfile): @@ -88,9 +81,13 @@ def from_pipfile(cls, name, pipfile): marker_strings = ["{0} {1}".format(k, pipfile[k]) for k in found_keys] if pipfile.get("markers"): marker_strings.append(pipfile.get("markers")) - markers = {} + markers = [] for marker in marker_strings: - marker_dict = cls.make_marker(marker) - if marker_dict: - markers.update(marker_dict) - return cls(**markers) + markers.append(marker) + marker = '' + try: + marker = cls.make_marker(" and ".join(markers)) + except RequirementError: + pass + else: + return marker diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index d49ce49416..177bd454e5 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -27,6 +27,7 @@ filter_none, optional_instance_of, split_markers_from_line, + parse_extras, ) from .._compat import ( Link, @@ -56,6 +57,7 @@ class NamedRequirement(BaseRequirement): name = attr.ib() version = attr.ib(validator=attr.validators.optional(validate_specifiers)) req = attr.ib() + extras = attr.ib(default=attr.Factory(list)) @req.default def get_requirement(self): @@ -114,6 +116,7 @@ class FileRequirement(BaseRequirement): path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) # : path to hit - without any of the VCS prefixes (like git+ / http+ / etc) editable = attr.ib(default=None) + extras = attr.ib(default=attr.Factory(list)) uri = attr.ib() link = attr.ib() name = attr.ib() @@ -501,6 +504,7 @@ def get_link(self): name=self.name, ref=self.ref, subdirectory=self.subdirectory, + extras=self.extras ) @name.default @@ -546,6 +550,8 @@ def get_requirement(self): req.vcs = self.vcs if self.ref and not req.revision: req.revision = self.ref + if self.extras and not req.extras: + req.extras = self.extras return req @classmethod @@ -553,7 +559,7 @@ def from_pipfile(cls, name, pipfile): creation_args = {} pipfile_keys = [ k - for k in ("ref", "vcs", "subdirectory", "path", "editable", "file", "uri") + for k in ("ref", "vcs", "subdirectory", "path", "editable", "file", "uri", "extras") + VCS_LIST if k in pipfile ] @@ -572,13 +578,18 @@ def from_pipfile(cls, name, pipfile): return cls(**creation_args) @classmethod - def from_line(cls, line, editable=None): + def from_line(cls, line, editable=None, extras=None): relpath = None if line.startswith("-e "): editable = True line = line.split(" ", 1)[1] vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) - name = link.egg_fragment + if not extras and link.egg_fragment: + name, extras = _strip_extras(link.egg_fragment) + if extras: + extras = parse_extras(extras) + else: + name = link.egg_fragment subdirectory = link.subdirectory_fragment ref = None if "@" in link.show_url and "@" in uri: @@ -594,6 +605,7 @@ def from_line(cls, line, editable=None): path=relpath or path, editable=editable, uri=uri, + extras=extras, ) @property @@ -623,7 +635,7 @@ def pipfile_part(self): pipfile_dict = attr.asdict(self, filter=filter_none).copy() if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) - name = pipfile_dict.pop("name") + name, _ = _strip_extras(pipfile_dict.pop("name")) return {name: pipfile_dict} @@ -659,7 +671,7 @@ def hashes_as_pip(self): @property def markers_as_pip(self): if self.markers: - return "; {0}".format(self.markers.replace('"', "'")) + return "; {0}".format(self.markers).replace('"', "'") return "" @@ -702,6 +714,8 @@ def from_line(cls, line): line = line.split(" ", 1)[1] if editable else line line, markers = split_markers_from_line(line) line, extras = _strip_extras(line) + if extras: + extras = parse_extras(extras) line = line.strip('"').strip("'").strip() line_with_prefix = "-e {0}".format(line) if editable else line vcs = None @@ -710,7 +724,7 @@ def from_line(cls, line): if is_installable_file(line) or (is_valid_url(line) and not is_vcs(line)): r = FileRequirement.from_line(line_with_prefix) elif is_vcs(line): - r = VCSRequirement.from_line(line_with_prefix) + r = VCSRequirement.from_line(line_with_prefix, extras=extras) vcs = r.vcs elif line == "." and not is_installable_file(line): raise RequirementError( @@ -727,14 +741,11 @@ def from_line(cls, line): version = line[spec_idx:] if not extras: name, extras = _strip_extras(name) + if extras: + extras = parse_extras(extras) if version: name = "{0}{1}".format(name, version) r = NamedRequirement.from_line(line) - if extras: - extras = first( - requirements.parse("fakepkg{0}".format(extras_to_string(extras))) - ).extras - r.req.extras = extras if markers: r.req.markers = markers args = { @@ -746,6 +757,10 @@ def from_line(cls, line): } if extras: args["extras"] = extras + r.req.extras = extras + r.extras = extras + elif r.extras: + args["extras"] = r.extras if hashes: args["hashes"] = hashes return cls(**args) @@ -764,11 +779,14 @@ def from_pipfile(cls, name, pipfile): r = FileRequirement.from_pipfile(name, pipfile) else: r = NamedRequirement.from_pipfile(name, pipfile) + markers = PipenvMarkers.from_pipfile(name, _pipfile) + if markers: + markers = str(markers) args = { "name": r.name, "vcs": vcs, "req": r, - "markers": PipenvMarkers.from_pipfile(name, _pipfile).line_part, + "markers": markers, "extras": _pipfile.get("extras"), "editable": _pipfile.get("editable", False), "index": _pipfile.get("index"), @@ -788,7 +806,7 @@ def as_line(self, sources=None): """ line = "{0}{1}{2}{3}{4}".format( self.req.line_part, - self.extras_as_pip, + self.extras_as_pip if not self.is_vcs else "", self.specifiers if self.specifiers else "", self.markers_as_pip, self.hashes_as_pip, diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 82309e9be9..44692399c7 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -32,6 +32,15 @@ def extras_to_string(extras): return "[{0}]".format(",".join(extras)) +def parse_extras(extras_str): + """Turn a string of extras into a parsed extras list""" + import requirements + extras = first( + requirements.parse("fakepkg{0}".format(extras_to_string(extras_str))) + ).extras + return extras + + def specs_to_string(specs): """Turn a list of specifier tuples into a string""" if specs: @@ -133,6 +142,7 @@ def validate_markers(instance, attr_, value): def validate_specifiers(instance, attr_, value): from packaging.specifiers import SpecifierSet, InvalidSpecifier + from packaging.markers import InvalidMarker if value == "": return True try: diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 2f278dff87..2b7781cee0 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -27,7 +27,7 @@ requests==2.19.1 idna==2.7 urllib3==1.23 certifi==2018.4.16 -requirementslib==1.0.9 +requirementslib==1.0.10 attrs==18.1.0 distlib==0.2.7 packaging==17.1