repository/legacy: calculate sha256 if unavailable

In some cases, legacy repositories might not provide a checksum as a url fragment or use a deprecated algorithm. In these scenarios, this change ensures that poetry downloads and calculates the sha256 checksum for the file. Resolves: #1631 #1553
python-poetry · Mar 21, 2021 · 1eef86f · 1eef86f
1 parent 37b36f4
commit 1eef86f
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 8 deletions.
diff --git a/poetry/repositories/legacy_repository.py b/poetry/repositories/legacy_repository.py
@@ -1,4 +1,5 @@
 import cgi
+import hashlib
 import re
 import urllib.parse
 import warnings
@@ -27,6 +28,8 @@
 from poetry.core.semver.version_range import VersionRange
 from poetry.locations import REPOSITORY_CACHE_DIR
 from poetry.utils.helpers import canonicalize_name
+from poetry.utils.helpers import download_file
+from poetry.utils.helpers import temporary_directory
 from poetry.utils.patterns import wheel_file_re
 
 from ..config.config import Config
@@ -374,10 +377,37 @@ def _get_release_info(self, name: str, version: str) -> dict:
             ):
                 urls["sdist"].append(link.url)
 
-            h = link.hash
-            if h:
-                h = link.hash_name + ":" + link.hash
-                files.append({"file": link.filename, "hash": h})
+            file_hash = "{}:{}".format(link.hash_name, link.hash) if link.hash else None
+
+            if not link.hash or (
+                link.hash_name not in ("sha256", "sha384", "sha512")
+                and hasattr(hashlib, link.hash_name)
+            ):
+                with temporary_directory() as temp_dir:
+                    filepath = Path(temp_dir) / link.filename
+                    self._download(link.url, str(filepath))
+
+                    known_hash = (
+                        getattr(hashlib, link.hash_name)() if link.hash_name else None
+                    )
+                    required_hash = hashlib.sha256()
+
+                    chunksize = 4096
+                    with filepath.open("rb") as f:
+                        while True:
+                            chunk = f.read(chunksize)
+                            if not chunk:
+                                break
+                            if known_hash:
+                                known_hash.update(chunk)
+                            required_hash.update(chunk)
+
+                    if not known_hash or known_hash.hexdigest() == link.hash:
+                        file_hash = "{}:{}".format(
+                            required_hash.name, required_hash.hexdigest()
+                        )
+
+            files.append({"file": link.filename, "hash": file_hash})
 
         data.files = files
 
@@ -415,3 +445,6 @@ def _get(self, endpoint: str) -> Optional[Page]:
             )
 
         return Page(response.url, response.content, response.headers)
+
+    def _download(self, url, dest):  # type: (str, str) -> None
+        return download_file(url, dest, session=self.session)
diff --git a/tests/repositories/fixtures/legacy/ipython.html b/tests/repositories/fixtures/legacy/ipython.html
@@ -5,10 +5,10 @@
   </head>
   <body>
     <h1>Links for ipython</h1>
-    <a href="https://files.pythonhosted.org/packages/52/19/aadde98d6bde1667d0bf431fb2d22451f880aaa373e0a241c7e7cb5815a0/ipython-5.7.0-py2-none-any.whl#sha256=707d1bbfc81e41e39ead1012af931bec6f80357b87e520af352e539cf5961dc0">ipython-5.7.0-py2-none-any.whl</a><br/>
+    <a href="https://files.pythonhosted.org/packages/52/19/aadde98d6bde1667d0bf431fb2d22451f880aaa373e0a241c7e7cb5815a0/ipython-5.7.0-py2-none-any.whl#md5=a10a802ef98da741cd6f4f6289d47ba7">ipython-5.7.0-py2-none-any.whl</a><br/>
     <a href="https://files.pythonhosted.org/packages/c7/b6/03e0b5b0972e6161d16c4cec8d41a20372bd0634f8cb4cc0c984b8a91db6/ipython-5.7.0-py3-none-any.whl#sha256=fc0464e68f9c65cd8c453474b4175432cc29ecb6c83775baedf6dbfcee9275ab">ipython-5.7.0-py3-none-any.whl</a><br/>
     <a href="https://files.pythonhosted.org/packages/3c/fd/559fead731a29eaa55cc235c8029807b2520976a937c30e9ee603f3bb566/ipython-5.7.0.tar.gz#sha256=8db43a7fb7619037c98626613ff08d03dda9d5d12c84814a4504c78c0da8323c">ipython-5.7.0.tar.gz</a><br/>
-    <a href="https://files.pythonhosted.org/packages/a9/2e/41dce4ed129057e05a555a7f9629aa2d5f81fdcd4d16568bc24b75a1d2c9/ipython-7.5.0-py3-none-any.whl#md5=dbdc53e3918f28fa335a173432402a00" data-requires-python="&gt;=3.5">ipython-7.5.0-py3-none-any.whl</a><br/>
+    <a href="https://files.pythonhosted.org/packages/a9/2e/41dce4ed129057e05a555a7f9629aa2d5f81fdcd4d16568bc24b75a1d2c9/ipython-7.5.0-py3-none-any.whl#md5=a10a802ef98da741cd6f4f6289d47ba8" data-requires-python="&gt;=3.5">ipython-7.5.0-py3-none-any.whl</a><br/>
     <a href="https://files.pythonhosted.org/packages/75/74/9b0ef91c8e356c907bb12297000951acb804583b54eeaddc342c5bad4d96/ipython-7.5.0.tar.gz#sha256=e840810029224b56cd0d9e7719dc3b39cf84d577f8ac686547c8ba7a06eeab26" data-requires-python="&gt;=3.5">ipython-7.5.0.tar.gz</a><br/>
   </body>
 </html>

diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py
@@ -273,7 +273,7 @@ def test_get_package_retrieves_non_sha256_hashes():
     expected = [
         {
             "file": "ipython-7.5.0-py3-none-any.whl",
-            "hash": "md5:dbdc53e3918f28fa335a173432402a00",
+            "hash": "sha256:78aea20b7991823f6a32d55f4e963a61590820e43f666ad95ad07c7f0c704efa",
         },
         {
             "file": "ipython-7.5.0.tar.gz",
@@ -284,12 +284,40 @@ def test_get_package_retrieves_non_sha256_hashes():
     assert expected == package.files
 
 
+def test_get_package_retrieves_non_sha256_hashes_mismatching_known_hash():
+    repo = MockRepository()
+
+    package = repo.package("ipython", "5.7.0")
+
+    expected = [
+        {
+            "file": "ipython-5.7.0-py2-none-any.whl",
+            "hash": "md5:a10a802ef98da741cd6f4f6289d47ba7",
+        },
+        {
+            "file": "ipython-5.7.0-py3-none-any.whl",
+            "hash": "sha256:fc0464e68f9c65cd8c453474b4175432cc29ecb6c83775baedf6dbfcee9275ab",
+        },
+        {
+            "file": "ipython-5.7.0.tar.gz",
+            "hash": "sha256:8db43a7fb7619037c98626613ff08d03dda9d5d12c84814a4504c78c0da8323c",
+        },
+    ]
+
+    assert expected == package.files
+
+
 def test_get_package_retrieves_packages_with_no_hashes():
     repo = MockRepository()
 
     package = repo.package("jupyter", "1.0.0")
 
-    assert [] == package.files
+    assert [
+        {
+            "file": "jupyter-1.0.0.tar.gz",
+            "hash": "sha256:d9dc4b3318f310e34c82951ea5d6683f67bed7def4b259fafbfe4f1beb1d8e5f",
+        }
+    ] == package.files
 
 
 class MockHttpRepository(LegacyRepository):