ngclient: allow compression in HTTP responses

This commit tries to deal with two interests:
* metadata is highly repetitive and compressible: allowing compression
  would be good
* there may be broken web servers (see
  https://github.com/pypa/pip/blob/404838abcca467648180b358598c597b74d568c9/src/pip/_internal/download.py#L842)
  that have problems with compression on already compressed target files

We can make things better for that first interest while we have no real
data for the second interest -- our current workarounds to avoid
compression are based on hearsay, not testing.

Now that individual fetchers are possible I suggest we simplify
ngclient and allow compression. As an example the pip Fetcher
could still use the pip response chunking code with all their
workarounds -- pip certainly has better capability to maintain
a mountain of workarounds and also has endless amounts of real-world
testing compared to python-tuf.

Details:
* Stop modifying Accept-Encoding (Requests default includes gzip)
* Don't use response.raw in RequestsFetcher as there is no need anymore
* Fix issue in test_session_get_timeout(): it's not mocking the error
  that requests really raises in this case

Fixes #1251

Signed-off-by: Jussi Kukkonen <jkukkonen@vmware.com>

Author: Jussi Kukkonen

tests/test_fetcher_ng.py

@@ -17,7 +17,6 @@
 from unittest.mock import Mock, patch
 
 import requests
-import urllib3.exceptions
 
 from tests import utils
 from tuf import unittest_toolbox
@@ -125,20 +124,22 @@ def test_http_error(self) -> None:
     def test_response_read_timeout(self, mock_session_get: Any) -> None:
         mock_response = Mock()
         attr = {
-            "raw.read.side_effect": urllib3.exceptions.ReadTimeoutError(
-                urllib3.HTTPConnectionPool("localhost"), "", "Read timed out."
+            "iter_content.side_effect": requests.exceptions.ConnectionError(
+                "Simulated timeout"
             )
         }
         mock_response.configure_mock(**attr)
         mock_session_get.return_value = mock_response
 
         with self.assertRaises(exceptions.SlowRetrievalError):
             next(self.fetcher.fetch(self.url))
-        mock_response.raw.read.assert_called_once()
+        mock_response.iter_content.assert_called_once()
 
     # Read/connect session timeout error
     @patch.object(
-        requests.Session, "get", side_effect=urllib3.exceptions.TimeoutError
+        requests.Session,
+        "get",
+        side_effect=requests.exceptions.Timeout("Simulated timeout"),
     )
     def test_session_get_timeout(self, mock_session_get: Any) -> None:
         with self.assertRaises(exceptions.SlowRetrievalError):

tuf/ngclient/_internal/requests_fetcher.py

@@ -11,7 +11,7 @@
 
 # Imports
 import requests
-import urllib3.exceptions
+import requests.exceptions
 
 import tuf
 from tuf.api import exceptions
@@ -80,7 +80,7 @@ def fetch(self, url: str) -> Iterator[bytes]:
             response = session.get(
                 url, stream=True, timeout=self.socket_timeout
             )
-        except urllib3.exceptions.TimeoutError as e:
+        except requests.exceptions.Timeout as e:
             raise exceptions.SlowRetrievalError from e
 
         # Check response status.
@@ -99,26 +99,12 @@ def _chunks(self, response: "requests.Response") -> Iterator[bytes]:
         download."""
 
         try:
-            while True:
-                # We download a fixed chunk of data in every round. This is
-                # so that we can defend against slow retrieval attacks.
-                # Furthermore, we do not wish to download an extremely
-                # large file in one shot.
-
-                # NOTE: This may not handle some servers adding a
-                # Content-Encoding header, which may cause urllib3 to
-                #  misbehave:
-                # https://github.com/pypa/pip/blob/404838abcca467648180b358598c597b74d568c9/src/pip/_internal/download.py#L547-L582
-                data = response.raw.read(self.chunk_size)
-
-                # We might have no more data to read, we signal
-                # that the download is complete.
-                if not data:
-                    break
-
+            for data in response.iter_content(self.chunk_size):
                 yield data
-
-        except urllib3.exceptions.ReadTimeoutError as e:
+        except (
+            requests.exceptions.ConnectionError,
+            requests.exceptions.Timeout,
+        ) as e:
             raise exceptions.SlowRetrievalError from e
 
         finally:
@@ -138,31 +124,17 @@ def _get_session(self, url: str) -> requests.Session:
         if not parsed_url.scheme or not parsed_url.hostname:
             raise exceptions.DownloadError("Failed to parse URL {url}")
 
-        session_index = parsed_url.scheme + "+" + parsed_url.hostname
+        session_index = f"{parsed_url.scheme}+{parsed_url.hostname}"
         session = self._sessions.get(session_index)
 
         if not session:
             session = requests.Session()
             self._sessions[session_index] = session
 
-            # Attach some default headers to every Session.
-            requests_user_agent = session.headers["User-Agent"]
-            # Follows the RFC: https://tools.ietf.org/html/rfc7231#section-5.5.3
-            tuf_user_agent = (
-                "tuf/" + tuf.__version__ + " " + requests_user_agent
-            )
-            session.headers.update(
-                {
-                    # Tell the server not to compress or modify anything.
-                    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding#Directives
-                    "Accept-Encoding": "identity",
-                    # The TUF user agent.
-                    "User-Agent": tuf_user_agent,
-                }
-            )
+            ua = f"tuf/{tuf.__version__} {session.headers['User-Agent']}"
+            session.headers["User-Agent"] = ua
 
             logger.debug("Made new session %s", session_index)
-
         else:
             logger.debug("Reusing session %s", session_index)