Fix artifact downloads for foreign platforms. (#1851)

Fix the `ArtifactDownloader` to re-use the tricks used for
`--style universal` locks to trick Pip into not rejecting artifacts
that are not compatible with the current interpreter.

FIxes #1849

pex/resolve/downloads.py

@@ -5,14 +5,15 @@
 
 from pex import hashing
 from pex.common import atomic_directory, safe_mkdir, safe_mkdtemp
-from pex.compatibility import urlparse
+from pex.compatibility import unquote, urlparse
 from pex.hashing import Sha256
 from pex.jobs import Job, Raise, SpawnedJob, execute_parallel
-from pex.pip.tool import PackageIndexConfiguration, Pip, get_pip
-from pex.resolve.locked_resolve import Artifact, FileArtifact
+from pex.pip.tool import PackageIndexConfiguration, get_pip
+from pex.resolve import locker
+from pex.resolve.locked_resolve import Artifact, FileArtifact, LockConfiguration, LockStyle
 from pex.resolve.resolved_requirement import Fingerprint, PartialArtifact
+from pex.resolve.resolvers import Resolver
 from pex.result import Error
-from pex.targets import LocalInterpreter, Target
 from pex.typing import TYPE_CHECKING
 from pex.variables import ENV
 
@@ -42,14 +43,10 @@ def get_downloads_dir(pex_root=None):
 
 @attr.s(frozen=True)
 class ArtifactDownloader(object):
+    resolver = attr.ib()  # type: Resolver
     package_index_configuration = attr.ib(
         default=PackageIndexConfiguration.create()
     )  # type: PackageIndexConfiguration
-    target = attr.ib(default=LocalInterpreter.create())  # type: Target
-    _pip = attr.ib(init=False)  # type: Pip
-
-    def __attrs_post_init__(self):
-        object.__setattr__(self, "_pip", get_pip(interpreter=self.target.get_interpreter()))
 
     @staticmethod
     def _fingerprint_and_move(path):
@@ -92,21 +89,40 @@ def _download(
                 url = credentialed_url
                 break
 
-        return self._pip.spawn_download_distributions(
+        # Although we don't actually need to observe the download, we do need to patch Pip to not
+        # care about wheel tags, environment markers or Requires-Python. The locker's download
+        # observer does just this for universal locks with no target system or requires python
+        # restrictions.
+        download_observer = locker.patch(
+            resolver=self.resolver,
+            lock_configuration=LockConfiguration(style=LockStyle.UNIVERSAL),
+            download_dir=download_dir,
+        )
+        return get_pip().spawn_download_distributions(
             download_dir=download_dir,
             requirements=[url],
             transitive=False,
-            target=self.target,
             package_index_configuration=self.package_index_configuration,
+            observer=download_observer,
         )
 
     def _download_and_fingerprint(self, url):
         # type: (str) -> SpawnedJob[FileArtifact]
         downloads = get_downloads_dir()
         download_dir = safe_mkdtemp(prefix="fingerprint_artifact.", dir=downloads)
-        temp_dest = os.path.join(
-            download_dir, os.path.basename(urlparse.unquote(urlparse.urlparse(url).path))
-        )
+
+        url_info = urlparse.urlparse(url)
+        src_file = urlparse.unquote(url_info.path)
+        temp_dest = os.path.join(download_dir, os.path.basename(src_file))
+
+        if url_info.scheme == "file":
+            shutil.copy(src_file, temp_dest)
+            return SpawnedJob.completed(
+                self._create_file_artifact(
+                    url, fingerprint=self._fingerprint_and_move(temp_dest), verified=True
+                )
+            )
+
         return SpawnedJob.and_then(
             self._download(url=url, download_dir=download_dir),
             result_func=lambda: self._create_file_artifact(
@@ -139,9 +155,19 @@ def download(
         digest,  # type: HintedDigest
     ):
         # type: (...) -> Union[str, Error]
-        try:
-            self._download(url=artifact.url, download_dir=dest_dir).wait()
-        except Job.Error as e:
-            return Error((e.stderr or str(e)).splitlines()[-1])
-        hashing.file_hash(os.path.join(dest_dir, artifact.filename), digest)
+        dest_file = os.path.join(dest_dir, artifact.filename)
+
+        url_info = urlparse.urlparse(artifact.url)
+        if url_info.scheme == "file":
+            src_file = unquote(url_info.path)
+            try:
+                shutil.copy(src_file, dest_file)
+            except (IOError, OSError) as e:
+                return Error(str(e))
+        else:
+            try:
+                self._download(url=artifact.url, download_dir=dest_dir).wait()
+            except Job.Error as e:
+                return Error((e.stderr or str(e)).splitlines()[-1])
+        hashing.file_hash(dest_file, digest)
         return artifact.filename

pex/resolve/lock_resolver.py

@@ -279,8 +279,8 @@ def resolve_from_lock(
             file_download_managers_by_target[target] = FileArtifactDownloadManager(
                 file_lock_style=file_lock_style,
                 downloader=ArtifactDownloader(
+                    resolver=resolver,
                     package_index_configuration=package_index_configuration,
-                    target=target,
                 ),
             )
 

pex/resolve/lockfile/create.py

@@ -201,7 +201,8 @@ def lock(self, downloaded):
                 resolved_requirements=resolved_requirements,
                 dist_metadatas=dist_metadatas_by_target[target],
                 fingerprinter=ArtifactDownloader(
-                    package_index_configuration=self.package_index_configuration, target=target
+                    resolver=self.resolver,
+                    package_index_configuration=self.package_index_configuration,
                 ),
                 platform_tag=None
                 if self.lock_configuration.style == LockStyle.UNIVERSAL

tests/integration/test_downloads.py

@@ -0,0 +1,84 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+import hashlib
+import os.path
+
+import pytest
+
+from pex.resolve.configured_resolver import ConfiguredResolver
+from pex.resolve.downloads import ArtifactDownloader
+from pex.resolve.locked_resolve import Artifact, FileArtifact
+from pex.resolve.resolved_requirement import Fingerprint, PartialArtifact
+from pex.resolve.resolver_configuration import PipConfiguration
+from pex.testing import IS_LINUX
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    pass
+
+
+def file_artifact(
+    url,  # type: str
+    sha256,  # type: str
+):
+    # type: (...) -> FileArtifact
+    artifact = Artifact.from_url(
+        url=url, fingerprint=Fingerprint(algorithm="sha256", hash=sha256), verified=True
+    )
+    assert isinstance(artifact, FileArtifact)
+    return artifact
+
+
+LINUX_ARTIFACT = file_artifact(
+    url=(
+        "https://files.pythonhosted.org/packages/6d/c6/"
+        "6a4e46802e8690d50ba6a56c7f79ac283e703fcfa0fdae8e41909c8cef1f/"
+        "psutil-5.9.1-cp310-cp310-"
+        "manylinux_2_12_x86_64"
+        ".manylinux2010_x86_64"
+        ".manylinux_2_17_x86_64"
+        ".manylinux2014_x86_64.whl"
+    ),
+    sha256="29a442e25fab1f4d05e2655bb1b8ab6887981838d22effa2396d584b740194de",
+)
+
+MAC_ARTIFACT = file_artifact(
+    url=(
+        "https://files.pythonhosted.org/packages/d1/16/"
+        "6239e76ab5d990dc7866bc22a80585f73421588d63b42884d607f5f815e2/"
+        "psutil-5.9.1-cp310-cp310-macosx_10_9_x86_64.whl"
+    ),
+    sha256="c7be9d7f5b0d206f0bbc3794b8e16fb7dbc53ec9e40bbe8787c6f2d38efcf6c9",
+)
+
+
+@pytest.fixture
+def downloader():
+    # type: () -> ArtifactDownloader
+    return ArtifactDownloader(ConfiguredResolver(PipConfiguration()))
+
+
+def test_issue_1849_download_foreign_artifact(
+    tmpdir,  # type: str
+    downloader,  # type: ArtifactDownloader
+):
+    # type: (...) -> None
+
+    foreign_artifact = MAC_ARTIFACT if IS_LINUX else LINUX_ARTIFACT
+
+    dest_dir = os.path.join(str(tmpdir), "dest_dir")
+    assert foreign_artifact.filename == downloader.download(
+        foreign_artifact, dest_dir=dest_dir, digest=hashlib.sha256()
+    )
+
+
+def test_issue_1849_fingerprint_foreign_artifact(
+    tmpdir,  # type: str
+    downloader,  # type: ArtifactDownloader
+):
+    # type: (...) -> None
+
+    expected_artifacts = [LINUX_ARTIFACT, MAC_ARTIFACT]
+    assert expected_artifacts == list(
+        downloader.fingerprint([PartialArtifact(artifact.url) for artifact in expected_artifacts])
+    )