Support requests.response.raw being a file-like object

Previously httpie relied on requests.models.Response.raw being
urllib3.HTTPResponse.  The Requests documentation specifies that
(requests.models.Response.raw)[https://docs.python-requests.org/en/master/api/#requests.Response.raw]
is a File-like object but allows for other types for internal use.

This change introduces graceful handling for scenarios when
requests.models.Response.raw is not urllib3.HTTPResponse. In such a scenario
httpie now falls back to extracting metadata from requests.models.Response
directly instead of direct access from protected protected members such as
response.raw._original_response. A side effect in this fallback procedure is
that we can no longer determine HTTP protocol version and report it as `??`.

This change is necessary to make it possible to implement TransportPlugins
without having to also needing to emulate internal behavior of urlib3 and
http.client.

Makefile

@@ -77,14 +77,20 @@ venv:
 
 test:
 	@echo $(H1)Running tests$(HEADER_EXTRA)$(H1END)
-	$(VENV_BIN)/python -m pytest $(COV) ./httpie $(COV) ./tests --doctest-modules --verbose ./httpie ./tests
+	$(VENV_BIN)/python -m pytest $(COV) ./httpie $(COV) ./tests --doctest-modules --verbose ./httpie ./tests $(MISSING)
 	@echo
 
 
 test-cover: COV=--cov
 test-cover: HEADER_EXTRA=' (with coverage)'
 test-cover: test
 
+test-cover-lines:
+test-cover-lines: COV=--cov
+test-cover-lines: HEADER_EXTRA=' (with missing lines coverage report)'
+test-cover-lines: MISSING=--cov-report term-missing
+test-cover-lines: test
+
 
 # test-all is meant to test everything — even this Makefile
 test-all: clean install test test-dist codestyle

httpie/client.py

@@ -104,10 +104,15 @@ def collect_messages(
                     **send_kwargs,
                 )
 
-            # noinspection PyProtectedMember
-            expired_cookies += get_expired_cookies(
-                headers=response.raw._original_response.msg._headers
-            )
+            if isinstance(response.raw, urllib3.HTTPResponse):
+                # noinspection PyProtectedMember
+                expired_cookies += get_expired_cookies(
+                    headers=response.raw._original_response.msg._headers
+                )
+            else:
+                expired_cookies += get_expired_cookies(
+                    headers=response.headers.items()
+                )
 
             response_count += 1
             if response.next:

httpie/models.py

@@ -1,5 +1,6 @@
 from typing import Iterable, Optional
 from urllib.parse import urlsplit
+import urllib3
 
 
 class HTTPMessage:
@@ -49,9 +50,8 @@ def iter_body(self, chunk_size=1):
     def iter_lines(self, chunk_size):
         return ((line, b'\n') for line in self._orig.iter_lines(chunk_size))
 
-    # noinspection PyProtectedMember
-    @property
-    def headers(self):
+    def _headers_http_client_http_message(self):
+        """ Extract header information from http.client.HTTPMessage """
         original = self._orig.raw._original_response
 
         version = {
@@ -61,12 +61,27 @@ def headers(self):
             20: '2',
         }[original.version]
 
-        status_line = f'HTTP/{version} {original.status} {original.reason}'
+        return version, original.status, original.reason, original.msg._headers
+
+    def _headers_requests_response(self):
+        """ Extract header information from Requests.models.Response """
+        version = "??"
+        return version, self._orig.status_code, self._orig.reason, self._orig.headers.items()
+
+
+    # noinspection PyProtectedMember
+    @property
+    def headers(self):
+        if isinstance(self._orig.raw, urllib3.HTTPResponse):
+            header_extract_method = self._headers_http_client_http_message
+        else:
+            header_extract_method = self._headers_requests_response
+        version, status, reason, http_headers = header_extract_method()
+
+        status_line = f'HTTP/{version} {status} {reason}'
         headers = [status_line]
-        # `original.msg` is a `http.client.HTTPMessage`
-        # `_headers` is a 2-tuple
         headers.extend(
-            f'{header[0]}: {header[1]}' for header in original.msg._headers)
+            ": ".join(header) for header in http_headers)
         return '\r\n'.join(headers)
 
     @property

tests/test_transport_plugin.py

@@ -0,0 +1,45 @@
+from io import BytesIO
+
+from requests.adapters import BaseAdapter
+from requests.models import Response
+from requests.utils import get_encoding_from_headers
+
+from httpie.plugins import TransportPlugin
+from httpie.plugins.registry import plugin_manager
+
+from .utils import HTTP_OK, http
+
+SCHEME = "http+fake"
+
+
+class FakeAdapter(BaseAdapter):
+    def send(self, request, **kwargs):
+        response = Response()
+        response.status_code = 200
+        response.reason = "OK"
+        response.headers = {
+            "Content-Type": "text/html; charset=UTF-8",
+        }
+        response.encoding = get_encoding_from_headers(response.headers)
+        response.raw = BytesIO(b"<!doctype html><html>Hello</html>")
+        return response
+
+
+class FakeTransportPlugin(TransportPlugin):
+    name = "Fake Transport"
+
+    prefix = SCHEME
+
+    def get_adapter(self):
+        return FakeAdapter()
+
+
+def test_transport_from_requests_response(httpbin):
+    plugin_manager.register(FakeTransportPlugin)
+    try:
+        r = http(f"{SCHEME}://example.com")
+        assert HTTP_OK in r
+        assert "Hello" in r
+        assert "Content-Type: text/html; charset=UTF-8" in r
+    finally:
+        plugin_manager.unregister(FakeTransportPlugin)