Merge pull request #654 from tiran/ignore-platform

feat: Add option to ignore wheel platform in resolver

Author: mergify[bot]

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 /work-dir*/
 /test-venv/
 /venv*/
+/.venv/
 /build/
 **/*.egg-info/
 /src/fromager/version.py

src/fromager/packagesettings.py

@@ -119,6 +119,7 @@ class ResolverDist(pydantic.BaseModel):
       sdist_server_url: https://pypi.org/simple/
       include_sdists: True
       include_wheels: False
+      ignore_platform: False
     """
 
     model_config = MODEL_CONFIG
@@ -132,6 +133,23 @@ class ResolverDist(pydantic.BaseModel):
     include_wheels: bool = False
     """Use wheels to resolve? (default: no)"""
 
+    ignore_platform: bool = False
+    """Ignore the platform when resolving with wheels? (default: no)
+
+    This option ignores the platform field (OS, CPU arch) when resolving with
+    *include_wheels* enabled.
+
+    .. versionadded:: 0.52
+    """
+
+    @pydantic.model_validator(mode="after")
+    def validate_ignore_platform(self) -> typing.Self:
+        if self.ignore_platform and not self.include_wheels:
+            raise ValueError(
+                "'ignore_platforms' has no effect without 'include_wheels'"
+            )
+        return self
+
 
 class DownloadSource(pydantic.BaseModel):
     """Package download source
@@ -696,6 +714,11 @@ def resolver_include_sdists(self) -> bool:
         """Include sdists when resolving package versions?"""
         return self._ps.resolver_dist.include_sdists
 
+    @property
+    def resolver_ignore_platform(self) -> bool:
+        """Ignore the platform when resolving with wheels?"""
+        return self._ps.resolver_dist.ignore_platform
+
     def build_dir(self, sdist_root_dir: pathlib.Path) -> pathlib.Path:
         """Build directory for package (e.g. subdirectory)"""
         build_dir = self._ps.build_dir

src/fromager/resolver.py

@@ -45,10 +45,17 @@
 
 PYTHON_VERSION = Version(python_version())
 DEBUG_RESOLVER = os.environ.get("DEBUG_RESOLVER", "")
-SUPPORTED_TAGS = set(sys_tags())
 PYPI_SERVER_URL = "https://pypi.org/simple"
 GITHUB_URL = "https://github.com"
 
+# all supported tags
+SUPPORTED_TAGS: frozenset[Tag] = frozenset(sys_tags())
+# same, but ignore the platform for 'ignore_platform' flag
+IGNORE_PLATFORM: str = "ignore"
+SUPPORTED_TAGS_IGNORE_PLATFORM: frozenset[Tag] = frozenset(
+    Tag(t.interpreter, t.abi, IGNORE_PLATFORM) for t in SUPPORTED_TAGS
+)
+
 
 def resolve(
     *,
@@ -58,6 +65,7 @@ def resolve(
     include_sdists: bool = True,
     include_wheels: bool = True,
     req_type: RequirementType | None = None,
+    ignore_platform: bool = False,
 ) -> tuple[str, Version]:
     # Create the (reusable) resolver.
     provider = overrides.find_and_invoke(
@@ -70,6 +78,7 @@ def resolve(
         include_wheels=include_wheels,
         sdist_server_url=sdist_server_url,
         req_type=req_type,
+        ignore_platform=ignore_platform,
     )
     return resolve_from_provider(provider, req)
 
@@ -81,6 +90,7 @@ def default_resolver_provider(
     include_sdists: bool,
     include_wheels: bool,
     req_type: RequirementType | None = None,
+    ignore_platform: bool = False,
 ) -> PyPIProvider | GenericProvider | GitHubTagProvider:
     """Lookup resolver provider to resolve package versions"""
     return PyPIProvider(
@@ -89,6 +99,7 @@ def default_resolver_provider(
         sdist_server_url=sdist_server_url,
         constraints=ctx.constraints,
         req_type=req_type,
+        ignore_platform=ignore_platform,
     )
 
 
@@ -144,6 +155,7 @@ def get_project_from_pypi(
     project: str,
     extras: typing.Iterable[str],
     sdist_server_url: str,
+    ignore_platform: bool = False,
 ) -> typing.Iterable[Candidate]:
     """Return candidates created from the project name and extras."""
     found_candidates: set[str] = set()
@@ -210,6 +222,15 @@ def get_project_from_pypi(
                 # FIXME: This doesn't take into account precedence of
                 # the supported tags for best fit.
                 matching_tags = SUPPORTED_TAGS.intersection(tags)
+                if not matching_tags and ignore_platform:
+                    if DEBUG_RESOLVER:
+                        logger.debug(f"{project}: ignoring platform for {filename}")
+                    ignore_platform_tags: frozenset[Tag] = frozenset(
+                        Tag(t.interpreter, t.abi, IGNORE_PLATFORM) for t in tags
+                    )
+                    matching_tags = SUPPORTED_TAGS_IGNORE_PLATFORM.intersection(
+                        ignore_platform_tags
+                    )
                 if not matching_tags:
                     if DEBUG_RESOLVER:
                         logger.debug(f"{project}: ignoring {filename} with tags {tags}")
@@ -272,13 +293,15 @@ def __init__(
         sdist_server_url: str = "https://pypi.org/simple/",
         constraints: Constraints | None = None,
         req_type: RequirementType | None = None,
+        ignore_platform: bool = False,
     ):
         super().__init__()
         self.include_sdists = include_sdists
         self.include_wheels = include_wheels
         self.sdist_server_url = sdist_server_url
         self.constraints = constraints or Constraints()
         self.req_type = req_type
+        self.ignore_platform = ignore_platform
 
     def identify(self, requirement_or_candidate: Requirement | Candidate) -> str:
         return canonicalize_name(requirement_or_candidate.name)
@@ -405,13 +428,15 @@ def __init__(
         sdist_server_url: str = "https://pypi.org/simple/",
         constraints: Constraints | None = None,
         req_type: RequirementType | None = None,
+        ignore_platform: bool = False,
     ):
         super().__init__(
             include_sdists=include_sdists,
             include_wheels=include_wheels,
             sdist_server_url=sdist_server_url,
             constraints=constraints,
             req_type=req_type,
+            ignore_platform=ignore_platform,
         )
 
     def get_cache(self) -> dict[str, list[Candidate]]:
@@ -456,7 +481,10 @@ def find_matches(
             # are added to the candidate at creation - we
             # treat candidates as immutable once created.
             for candidate in get_project_from_pypi(
-                identifier, set(), self.sdist_server_url
+                identifier,
+                set(),
+                self.sdist_server_url,
+                self.ignore_platform,
             ):
                 if self.validate_candidate(
                     identifier, requirements, incompatibilities, candidate

src/fromager/sources.py

@@ -165,6 +165,7 @@ def default_resolve_source(
         include_sdists=pbi.resolver_include_sdists,
         include_wheels=pbi.resolver_include_wheels,
         req_type=req_type,
+        ignore_platform=pbi.resolver_ignore_platform,
     )
     return url, version
 

src/fromager/wheels.py

@@ -428,6 +428,8 @@ def resolve_prebuilt_wheel(
                 include_sdists=False,
                 include_wheels=True,
                 req_type=req_type,
+                # pre-built wheels must match platform
+                ignore_platform=False,
             )
         except Exception:
             continue

tests/test_packagesettings.py

@@ -15,6 +15,7 @@
     GitOptions,
     Package,
     PackageSettings,
+    ResolverDist,
     SettingsFile,
     Variant,
     substitute_template,
@@ -67,8 +68,9 @@
     },
     "resolver_dist": {
         "include_sdists": True,
-        "include_wheels": False,
+        "include_wheels": True,
         "sdist_server_url": "https://sdist.test/egg",
+        "ignore_platform": True,
     },
     "variants": {
         "cpu": {
@@ -119,6 +121,7 @@
         "sdist_server_url": None,
         "include_sdists": True,
         "include_wheels": False,
+        "ignore_platform": False,
     },
     "variants": {},
 }
@@ -266,7 +269,8 @@ def test_pbi_test_pkg(testdata_context: context.WorkContext) -> None:
         == "test-pkg-1.0.2.tar.gz"
     )
     assert pbi.resolver_include_sdists is True
-    assert pbi.resolver_include_wheels is False
+    assert pbi.resolver_include_wheels is True
+    assert pbi.resolver_ignore_platform is True
     assert (
         pbi.resolver_sdist_server_url("https://pypi.org/simple")
         == "https://sdist.test/egg"
@@ -630,3 +634,8 @@ def test_package_build_info_exclusive_build(testdata_context: context.WorkContex
 
     custom_pbi = settings.package_build_info("exclusive-pkg")
     assert custom_pbi.exclusive_build is True
+
+
+def test_resolver_dist_validator():
+    with pytest.raises(pydantic.ValidationError):
+        ResolverDist(include_wheels=False, ignore_platform=True)

tests/test_resolver.py

@@ -267,6 +267,64 @@ def test_provider_constraint_match():
         assert str(candidate.version) == "1.2.2"
 
 
+_ignore_platform_simple_response = """
+<!DOCTYPE html>
+<html>
+<head>
+<meta name="pypi:repository-version" content="1.1">
+<title>Links for fromager</title>
+</head>
+<body>
+<h1>Links for fromager</h1>
+<a href="https://files.pythonhosted.org/packages/7c/06/620610984c2794ef55c4257c77211b7a625431b380880c524c2f6bc264b1/fromager-0.51.0-cp311-abi3-manylinux_2_28_plan9.whl" >fromager-0.51.0-cp311-abi3-manylinux_2_28_plan9.whl</a>
+<br />
+<a href="https://files.pythonhosted.org/packages/7c/06/620610984c2794ef55c4257c77211b7a625431b380880c524c2f6bc264b1/fromager-0.51.0-cp3000-abi3-manylinux_2_28_plan9.whl" >fromager-0.51.0-cp3000-abi3-manylinux_2_28_plan9.whl</a>
+</body>
+</html>
+"""
+
+
+def test_provider_platform_mismatch():
+    constraint = constraints.Constraints()
+    with requests_mock.Mocker() as r:
+        r.get(
+            "https://pypi.org/simple/fromager/",
+            text=_ignore_platform_simple_response,
+        )
+
+        provider = resolver.PyPIProvider(include_wheels=True, constraints=constraint)
+        reporter = resolvelib.BaseReporter()
+        rslvr = resolvelib.Resolver(provider, reporter)
+
+        with pytest.raises(resolvelib.resolvers.exceptions.ResolverException):
+            rslvr.resolve([Requirement("fromager")])
+
+
+def test_provider_ignore_platform():
+    constraint = constraints.Constraints()
+    with requests_mock.Mocker() as r:
+        r.get(
+            "https://pypi.org/simple/fromager/",
+            text=_ignore_platform_simple_response,
+        )
+
+        provider = resolver.PyPIProvider(
+            include_wheels=True, constraints=constraint, ignore_platform=True
+        )
+        reporter = resolvelib.BaseReporter()
+        rslvr = resolvelib.Resolver(provider, reporter)
+
+        result = rslvr.resolve([Requirement("fromager")])
+        assert "fromager" in result.mapping
+
+        candidate = result.mapping["fromager"]
+        assert (
+            candidate.url
+            == "https://files.pythonhosted.org/packages/7c/06/620610984c2794ef55c4257c77211b7a625431b380880c524c2f6bc264b1/fromager-0.51.0-cp311-abi3-manylinux_2_28_plan9.whl"
+        )
+        assert str(candidate.version) == "0.51.0"
+
+
 _github_fromager_repo_response = """
 {
   "id": 808690091,

tests/test_sources.py

@@ -40,8 +40,9 @@ def test_default_download_source_from_settings(
         req=req,
         sdist_server_url=sdist_server_url,
         include_sdists=True,
-        include_wheels=False,
+        include_wheels=True,
         req_type=None,
+        ignore_platform=True,
     )
 
     sources.default_download_source(
@@ -86,6 +87,7 @@ def test_default_download_source_with_predefined_resolve_dist(
         include_sdists=False,
         include_wheels=True,
         req_type=None,
+        ignore_platform=False,
     )
 
 

tests/testdata/context/overrides/settings/test_pkg.yaml

@@ -33,7 +33,8 @@ project_override:
 resolver_dist:
     sdist_server_url: https://sdist.test/egg
     include_sdists: true
-    include_wheels: false
+    include_wheels: true
+    ignore_platform: true
 variants:
     cpu:
         env: