Skip to content

Commit 2d5977e

Browse files
committed
[Feat]: Add support for PEP 658 Metadata
This commit is my attempt to add support for PEP 658 which looks for Metadata URL while resolution I have attempted to handle backwards compatibility for packages that do not support PEP 658 Further, this commit also adds unit tests to test this functionality Fixes #360 Signed-off-by: Rohan Devasthale <rdevasth@redhat.com>
1 parent 8ae931f commit 2d5977e

File tree

4 files changed

+241
-9
lines changed

4 files changed

+241
-9
lines changed

src/fromager/candidate.py

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import logging
12
import typing
23
from email.message import EmailMessage, Message
34
from email.parser import BytesParser
@@ -11,6 +12,8 @@
1112

1213
from .request_session import session
1314

15+
logger = logging.getLogger(__name__)
16+
1417
# fix for runtime errors caused by inheriting classes that are generic in stubs but not runtime
1518
# https://mypy.readthedocs.io/en/latest/runtime_troubles.html#using-classes-that-are-generic-in-stubs-but-not-at-runtime
1619
if TYPE_CHECKING:
@@ -28,13 +31,15 @@ def __init__(
2831
extras: typing.Iterable[str] | None = None,
2932
is_sdist: bool | None = None,
3033
build_tag: BuildTag = (),
34+
metadata_url: str | None = None,
3135
):
3236
self.name = canonicalize_name(name)
3337
self.version = version
3438
self.url = url
3539
self.extras = extras
3640
self.is_sdist = is_sdist
3741
self.build_tag = build_tag
42+
self.metadata_url = metadata_url
3843

3944
self._metadata: Metadata | None = None
4045
self._dependencies: list[Requirement] | None = None
@@ -47,7 +52,7 @@ def __repr__(self) -> str:
4752
@property
4853
def metadata(self) -> Metadata:
4954
if self._metadata is None:
50-
self._metadata = get_metadata_for_wheel(self.url)
55+
self._metadata = get_metadata_for_wheel(self.url, self.metadata_url)
5156
return self._metadata
5257

5358
def _get_dependencies(self) -> typing.Iterable[Requirement]:
@@ -74,7 +79,40 @@ def requires_python(self) -> str | None:
7479
return self.metadata.get("Requires-Python")
7580

7681

77-
def get_metadata_for_wheel(url: str) -> Metadata:
82+
def get_metadata_for_wheel(url: str, metadata_url: str | None = None) -> Metadata:
83+
"""
84+
Get metadata for a wheel, supporting PEP 658 metadata endpoints.
85+
86+
Args:
87+
url: URL of the wheel file
88+
metadata_url: Optional URL of the metadata file (PEP 658)
89+
90+
Returns:
91+
Parsed metadata as a Message object
92+
"""
93+
# Try PEP 658 metadata endpoint first if available
94+
if metadata_url:
95+
try:
96+
logger.debug(
97+
f"Attempting to fetch metadata from PEP 658 endpoint: {metadata_url}"
98+
)
99+
response = session.get(metadata_url)
100+
response.raise_for_status()
101+
102+
# Parse metadata directly from the response content
103+
p = BytesParser()
104+
metadata = p.parse(BytesIO(response.content), headersonly=True)
105+
logger.debug(f"Successfully retrieved metadata via PEP 658 for {url}")
106+
return metadata
107+
108+
except Exception as e:
109+
logger.debug(f"Failed to fetch PEP 658 metadata from {metadata_url}: {e}")
110+
logger.debug(
111+
"Falling back to downloading full wheel for metadata extraction"
112+
)
113+
114+
# Fallback to existing method: download wheel and extract metadata
115+
logger.debug(f"Downloading full wheel to extract metadata: {url}")
78116
data = session.get(url).content
79117
with ZipFile(BytesIO(data)) as z:
80118
for n in z.namelist():

src/fromager/resolver.py

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -138,9 +138,9 @@ def resolve_from_provider(
138138
rslvr: resolvelib.Resolver = resolvelib.Resolver(provider, reporter)
139139
try:
140140
result = rslvr.resolve([req])
141-
except resolvelib.resolvers.exceptions.ResolverException as err:
141+
except resolvelib.resolvers.ResolverException as err:
142142
constraint = provider.constraints.get_constraint(req.name)
143-
raise resolvelib.resolvers.exceptions.ResolverException(
143+
raise resolvelib.resolvers.ResolverException(
144144
f"Unable to resolve requirement specifier {req} with constraint {constraint}"
145145
) from err
146146
# resolvelib actually just returns one candidate per requirement.
@@ -180,12 +180,26 @@ def get_project_from_pypi(
180180
for i in doc.findall(".//a"):
181181
candidate_url = urljoin(simple_index_url, i.attrib["href"])
182182
py_req = i.attrib.get("data-requires-python")
183+
# PEP 658: Check for metadata availability (PEP 714 data-core-metadata first)
184+
dist_info_metadata = i.attrib.get("data-core-metadata") or i.attrib.get(
185+
"data-dist-info-metadata"
186+
)
183187
path = urlparse(candidate_url).path
184188
# file names are URL quoted, "1.0%2Blocal" -> "1.0+local"
185189
filename = unquote(path.rsplit("/", 1)[-1])
186190
found_candidates.add(filename)
187191
if DEBUG_RESOLVER:
188192
logger.debug("%s: candidate %r -> %r", project, candidate_url, filename)
193+
194+
# Construct metadata URL if PEP 658 metadata is available
195+
metadata_url = None
196+
if dist_info_metadata:
197+
# PEP 658: metadata is available at {file_url}.metadata
198+
metadata_url = candidate_url + ".metadata"
199+
if DEBUG_RESOLVER:
200+
logger.debug(
201+
"%s: PEP 658 metadata available at %s", project, metadata_url
202+
)
189203
# Skip items that need a different Python version
190204
if py_req:
191205
try:
@@ -264,6 +278,7 @@ def get_project_from_pypi(
264278
extras=extras,
265279
is_sdist=is_sdist,
266280
build_tag=build_tag,
281+
metadata_url=metadata_url,
267282
)
268283
if DEBUG_RESOLVER:
269284
logger.debug(
@@ -498,14 +513,14 @@ def find_matches(
498513
# resolve.
499514
r = next(iter(requirements[identifier]))
500515
if self.include_sdists and self.include_wheels:
501-
raise resolvelib.resolvers.exceptions.ResolverException(
516+
raise resolvelib.resolvers.ResolverException(
502517
f"found no match for {r}, any file type, in cache or at {self.sdist_server_url}"
503518
)
504519
elif self.include_sdists:
505-
raise resolvelib.resolvers.exceptions.ResolverException(
520+
raise resolvelib.resolvers.ResolverException(
506521
f"found no match for {r}, limiting search to sdists, in cache or at {self.sdist_server_url}"
507522
)
508-
raise resolvelib.resolvers.exceptions.ResolverException(
523+
raise resolvelib.resolvers.ResolverException(
509524
f"found no match for {r}, limiting search to wheels, in cache or at {self.sdist_server_url}"
510525
)
511526
return sorted(candidates, key=attrgetter("version", "build_tag"), reverse=True)

tests/test_pep658_support.py

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
"""Tests for PEP 658 metadata support."""
2+
3+
from unittest.mock import Mock, patch
4+
5+
from packaging.version import Version
6+
7+
from fromager.candidate import Candidate, get_metadata_for_wheel
8+
9+
10+
class TestPEP658Support:
11+
"""Test PEP 658 metadata support in fromager."""
12+
13+
def test_candidate_with_metadata_url(self):
14+
"""Test that Candidate can be created with a metadata URL."""
15+
candidate = Candidate(
16+
name="test-package",
17+
version=Version("1.0.0"),
18+
url="https://example.com/test-package-1.0.0-py3-none-any.whl",
19+
metadata_url="https://example.com/test-package-1.0.0-py3-none-any.whl.metadata",
20+
)
21+
22+
assert (
23+
candidate.metadata_url
24+
== "https://example.com/test-package-1.0.0-py3-none-any.whl.metadata"
25+
)
26+
27+
def test_candidate_without_metadata_url(self):
28+
"""Test that Candidate works without metadata URL (legacy behavior)."""
29+
candidate = Candidate(
30+
name="test-package",
31+
version=Version("1.0.0"),
32+
url="https://example.com/test-package-1.0.0-py3-none-any.whl",
33+
)
34+
35+
assert candidate.metadata_url is None
36+
37+
@patch("fromager.candidate.session")
38+
def test_get_metadata_with_pep658_success(self, mock_session):
39+
"""Test successful metadata retrieval via PEP 658 endpoint."""
40+
# Mock the metadata response
41+
mock_response = Mock()
42+
mock_response.content = b"""Metadata-Version: 2.1
43+
Name: test-package
44+
Version: 1.0.0
45+
Summary: A test package
46+
Requires-Dist: requests >= 2.0.0
47+
"""
48+
mock_response.raise_for_status.return_value = None
49+
mock_session.get.return_value = mock_response
50+
51+
wheel_url = "https://example.com/test-package-1.0.0-py3-none-any.whl"
52+
metadata_url = (
53+
"https://example.com/test-package-1.0.0-py3-none-any.whl.metadata"
54+
)
55+
56+
metadata = get_metadata_for_wheel(wheel_url, metadata_url)
57+
58+
# Verify the metadata was parsed correctly
59+
assert metadata["Name"] == "test-package"
60+
assert metadata["Version"] == "1.0.0"
61+
assert metadata["Summary"] == "A test package"
62+
assert "requests >= 2.0.0" in metadata.get_all("Requires-Dist", [])
63+
64+
# Verify only the metadata URL was called, not the wheel URL
65+
mock_session.get.assert_called_once_with(metadata_url)
66+
67+
@patch("fromager.candidate.session")
68+
def test_get_metadata_pep658_fallback_behavior(self, mock_session):
69+
"""Test that PEP 658 is tried first, then falls back to wheel download."""
70+
# Mock that metadata URL fails, then wheel URL succeeds
71+
responses = []
72+
73+
def side_effect(url):
74+
if url.endswith(".metadata"):
75+
# First call - metadata request fails
76+
mock_response = Mock()
77+
mock_response.raise_for_status.side_effect = Exception("404 Not Found")
78+
responses.append(("metadata", url))
79+
return mock_response
80+
else:
81+
# Second call - wheel request
82+
responses.append(("wheel", url))
83+
raise Exception("Wheel parsing intentionally mocked to fail")
84+
85+
mock_session.get.side_effect = side_effect
86+
87+
wheel_url = "https://example.com/test-package-1.0.0-py3-none-any.whl"
88+
metadata_url = (
89+
"https://example.com/test-package-1.0.0-py3-none-any.whl.metadata"
90+
)
91+
92+
# This should raise an exception during wheel parsing, but we can verify the order
93+
try:
94+
get_metadata_for_wheel(wheel_url, metadata_url)
95+
except Exception:
96+
pass # Expected to fail during wheel parsing mock
97+
98+
# Verify that both URLs were called in the correct order
99+
assert len(responses) == 2
100+
assert responses[0] == ("metadata", metadata_url)
101+
assert responses[1] == ("wheel", wheel_url)
102+
assert mock_session.get.call_count == 2
103+
104+
@patch("fromager.candidate.session")
105+
def test_get_metadata_without_pep658_behavior(self, mock_session):
106+
"""Test that without PEP 658 metadata URL, only wheel URL is called."""
107+
# Mock wheel request
108+
responses = []
109+
110+
def side_effect(url):
111+
responses.append(("wheel", url))
112+
raise Exception("Wheel parsing intentionally mocked to fail")
113+
114+
mock_session.get.side_effect = side_effect
115+
116+
wheel_url = "https://example.com/test-package-1.0.0-py3-none-any.whl"
117+
118+
# This should raise an exception during wheel parsing, but we can verify the behavior
119+
try:
120+
get_metadata_for_wheel(wheel_url, metadata_url=None)
121+
except Exception:
122+
pass # Expected to fail during wheel parsing mock
123+
124+
# Verify that only the wheel URL was called
125+
assert len(responses) == 1
126+
assert responses[0] == ("wheel", wheel_url)
127+
mock_session.get.assert_called_once_with(wheel_url)
128+
129+
def test_candidate_repr_with_metadata_url(self):
130+
"""Test that Candidate representation includes metadata URL info."""
131+
candidate = Candidate(
132+
name="test-package",
133+
version=Version("1.0.0"),
134+
url="https://example.com/test-package-1.0.0-py3-none-any.whl",
135+
metadata_url="https://example.com/test-package-1.0.0-py3-none-any.whl.metadata",
136+
)
137+
138+
# The candidate should have the metadata URL attribute
139+
assert hasattr(candidate, "metadata_url")
140+
assert candidate.metadata_url is not None
141+
142+
def test_metadata_url_construction(self):
143+
"""Test that metadata URLs are constructed correctly."""
144+
base_url = (
145+
"https://pypi.org/simple/test-package/test-package-1.0.0-py3-none-any.whl"
146+
)
147+
expected_metadata_url = base_url + ".metadata"
148+
149+
# This tests the expected pattern for PEP 658 metadata URLs
150+
assert expected_metadata_url.endswith(".whl.metadata")
151+
assert expected_metadata_url.startswith("https://")
152+
153+
def test_pep658_integration_with_resolver(self):
154+
"""Test that PEP 658 metadata URLs are properly handled by the candidate system."""
155+
# Test the basic integration of metadata URLs with candidates
156+
candidate_with_metadata = Candidate(
157+
name="test-package",
158+
version=Version("1.0.0"),
159+
url="https://example.com/test.whl",
160+
metadata_url="https://example.com/test.whl.metadata",
161+
)
162+
163+
candidate_without_metadata = Candidate(
164+
name="test-package",
165+
version=Version("1.0.0"),
166+
url="https://example.com/test.whl",
167+
)
168+
169+
# Verify PEP 658 metadata URL handling
170+
assert (
171+
candidate_with_metadata.metadata_url
172+
== "https://example.com/test.whl.metadata"
173+
)
174+
assert candidate_without_metadata.metadata_url is None
175+
176+
# Both should have the same basic properties
177+
assert candidate_with_metadata.name == candidate_without_metadata.name
178+
assert candidate_with_metadata.version == candidate_without_metadata.version
179+
assert candidate_with_metadata.url == candidate_without_metadata.url

tests/test_resolver.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ def test_provider_constraint_mismatch():
239239
reporter = resolvelib.BaseReporter()
240240
rslvr = resolvelib.Resolver(provider, reporter)
241241

242-
with pytest.raises(resolvelib.resolvers.exceptions.ResolverException):
242+
with pytest.raises(resolvelib.resolvers.ResolverException):
243243
rslvr.resolve([Requirement("hydra-core")])
244244

245245

@@ -296,7 +296,7 @@ def test_provider_platform_mismatch():
296296
reporter = resolvelib.BaseReporter()
297297
rslvr = resolvelib.Resolver(provider, reporter)
298298

299-
with pytest.raises(resolvelib.resolvers.exceptions.ResolverException):
299+
with pytest.raises(resolvelib.resolvers.ResolverException):
300300
rslvr.resolve([Requirement("fromager")])
301301

302302

0 commit comments

Comments
 (0)