Replace response.iter_content by response.raw in RequestsFetcher._chunks.

This prevents requests/urllib3 from automatically decoding files with Content-Encoding gzip or deflate.

Author: dennisvang

tests/test_fetcher_ng.py

@@ -17,6 +17,7 @@
 from unittest.mock import Mock, patch
 
 import requests
+from requests.models import ReadTimeoutError  # actually from urllib3
 
 from tests import utils
 from tuf.api import exceptions
@@ -109,18 +110,20 @@ def test_http_error(self) -> None:
     # Response read timeout error
     @patch.object(requests.Session, "get")
     def test_response_read_timeout(self, mock_session_get: Any) -> None:
-        mock_response = Mock()
+        # from urllib3.connectionpool import ConnectionPool
+        # dummy_pool = ConnectionPool("dummy")
+        mock_response = Mock(raw=Mock())
         attr = {
-            "iter_content.side_effect": requests.exceptions.ConnectionError(
-                "Simulated timeout"
+            "read.side_effect": ReadTimeoutError(
+                None, None, "Simulated timeout"
             )
         }
-        mock_response.configure_mock(**attr)
+        mock_response.raw.configure_mock(**attr)
         mock_session_get.return_value = mock_response
 
         with self.assertRaises(exceptions.SlowRetrievalError):
             next(self.fetcher.fetch(self.url))
-        mock_response.iter_content.assert_called_once()
+        mock_response.raw.read.assert_called_once()
 
     # Read/connect session timeout error
     @patch.object(
@@ -171,6 +174,14 @@ def test_download_file_length_mismatch(self) -> Iterator[Any]:
             # context manager and returns Iterator[IO]
             yield self.fetcher.download_file(self.url, self.file_length - 4)
 
+    # Download a file with Content-Encoding gzip (or deflate)
+    def test_download_file_content_encoding(self):
+        # todo:
+        #  - enable custom headers in simple_server.py, so we can return a
+        #    file with Content-Encoding header
+        #  - serve a gzipped file (in TestFetcher.setUpClass?)
+        pass
+
 
 # Run unit test.
 if __name__ == "__main__":

tuf/ngclient/_internal/requests_fetcher.py

@@ -11,6 +11,7 @@
 
 # Imports
 import requests
+from requests.models import ReadTimeoutError  # todo: import directly from urllib3?
 
 import tuf
 from tuf.api import exceptions
@@ -19,6 +20,7 @@
 # Globals
 logger = logging.getLogger(__name__)
 
+
 # Classes
 class RequestsFetcher(FetcherInterface):
     """An implementation of ``FetcherInterface`` based on the requests library.
@@ -96,11 +98,20 @@ def _chunks(self, response: "requests.Response") -> Iterator[bytes]:
         download."""
 
         try:
-            for data in response.iter_content(self.chunk_size):
+            while True:
+                # Requests already calls `urlopen(..., decode_content=False)`,
+                # but we explicitly disable decode_content here, to be safe.
+                data = response.raw.read(
+                    amt=self.chunk_size, decode_content=False
+                )
+                if not data:
+                    break
                 yield data
         except (
-            requests.exceptions.ConnectionError,
-            requests.exceptions.Timeout,
+            # Catch urllib3 exceptions instead of requests exceptions.
+            ReadTimeoutError,
+            # todo: Is this sufficient? Do we need to catch IncompleteRead too?
+            #  https://github.com/urllib3/urllib3/blob/43c372f6eb9f9f94848c7b7645ad19ebf6c047c5/src/urllib3/response.py#L734
         ) as e:
             raise exceptions.SlowRetrievalError from e