Accept Brotli as encoding too, if supported (psf#4525)

by the urllib3 version (>= 1.23.1) and brotli package being
present

.travis.yml

@@ -21,6 +21,11 @@ jobs:
       dist: xenial
     - stage: test
       python: '3.8'
+    - stage: test
+      python: '3.8'
+      name: '3.8 without Brotli'
+      before_script:
+        - pip uninstall brotli
     - stage: coverage
       python: '3.6'
       script: codecov

Pipfile

@@ -12,7 +12,7 @@ detox = "*"
 httpbin = ">=0.7.0"
 more-itertools = "<6.0"
 pysocks = "*"
-pytest = ">=2.8.0,<=3.10.1"
+pytest = ">=3"
 pytest-httpbin = ">=0.0.7,<1.0"
 pytest-mock = "*"
 pytest-cov = "*"
@@ -22,4 +22,4 @@ sphinx = "<=1.5.5"
 tox = "*"
 
 [packages]
-"requests" = {path = ".", editable = true, extras = ["socks"]}
+"requests" = {path = ".", editable = true, extras = ["socks", "brotli"]}

docs/community/faq.rst

@@ -10,7 +10,7 @@ This part of the documentation answers common questions about Requests.
 Encoded Data?
 -------------
 
-Requests automatically decompresses gzip-encoded responses, and does
+Requests automatically decompresses gzip-, deflate- and brotli-encoded[#]_ responses, and does
 its best to decode response content to unicode when possible.
 
 You can get direct access to the raw response (and even the socket),
@@ -91,3 +91,6 @@ For information on using SNI with Requests on Python < 2.7.9 refer to this
 .. _`Server-Name-Indication`: https://en.wikipedia.org/wiki/Server_Name_Indication
 .. _`virtual hosting`: https://en.wikipedia.org/wiki/Virtual_hosting
 .. _`Stack Overflow answer`: https://stackoverflow.com/questions/18578439/using-requests-with-tls-doesnt-give-sni-support/18579484#18579484
+
+
+  [#] for Brotli support you need to have `urllib3` >= 1.2.1 and `brotli` packages installed

docs/user/advanced.rst

@@ -113,8 +113,8 @@ However, if we want to get the headers we sent the server, we simply access the
 request, and then the request's headers::
 
     >>> r.request.headers
-    {'Accept-Encoding': 'identity, deflate, compress, gzip',
-    'Accept': '*/*', 'User-Agent': 'python-requests/1.2.0'}
+    {'Accept-Encoding': 'br, gzip, deflate',
+    'Accept': '*/*', 'User-Agent': 'python-requests/2.25.0'}
 
 .. _prepared-requests:
 

docs/user/quickstart.rst

@@ -128,7 +128,7 @@ You can also access the response body as bytes, for non-text requests::
     >>> r.content
     b'[{"repository":{"open_issues":0,"url":"https://github.com/...
 
-The ``gzip`` and ``deflate`` transfer-encodings are automatically decoded for you.
+The ``gzip``, ``deflate`` and ``brotli``[#]_ transfer-encodings are automatically decoded for you.
 
 For example, to create an image from binary data returned by a request, you can
 use the following code::
@@ -559,7 +559,8 @@ All exceptions that Requests explicitly raises inherit from
 
 -----------------------
 
-Ready for more? Check out the :ref:`advanced <advanced>` section.
+  [#] for Brotli support you need to have `urllib3` >= 1.2.1 and `brotli` packages installed
 
+Ready for more? Check out the :ref:`advanced <advanced>` section.
 
 If you're on the job market, consider taking `this programming quiz <https://triplebyte.com/a/b1i2FB8/requests-docs-1>`_. A substantial donation will be made to this project, if you find a job through this platform.

requests/utils.py

@@ -20,6 +20,7 @@
 import warnings
 import zipfile
 from collections import OrderedDict
+import urllib3
 
 from .__version__ import __version__
 from . import certs
@@ -804,13 +805,44 @@ def default_user_agent(name="python-requests"):
     return '%s/%s' % (name, __version__)
 
 
+def brotli_supported():
+    """
+    Returns whether Brotli compression is supported
+
+    :rtype: bool
+    """
+
+    # urllib >= 1.25.1 includes brotli support
+    major, minor, patch = urllib3.__version__.split('.')  # noqa: F811
+    major, minor, patch = int(major), int(minor), int(patch)
+    urllib3_with_brotli = (major == 1 and ((minor == 25 and patch >= 1) or (minor >= 26))) \
+        or major >= 2
+
+    # pybrotli is an extra package required by urllib3 for brotli support
+    try:
+        import brotli
+    except ImportError:
+        brotli = None
+
+    return urllib3_with_brotli and brotli
+
+
+BROTLI_SUPPORTED = brotli_supported()
+
+
 def default_headers():
     """
     :rtype: requests.structures.CaseInsensitiveDict
     """
+
+    if BROTLI_SUPPORTED:
+        accepted_encodings = ('br', 'gzip', 'deflate')
+    else:
+        accepted_encodings = ('gzip', 'deflate')
+
     return CaseInsensitiveDict({
         'User-Agent': default_user_agent(),
-        'Accept-Encoding': ', '.join(('gzip', 'deflate')),
+        'Accept-Encoding': ', '.join(accepted_encodings),
         'Accept': '*/*',
         'Connection': 'keep-alive',
     })

setup.py

@@ -54,6 +54,7 @@ def run_tests(self):
     'pytest-mock',
     'pytest-xdist',
     'PySocks>=1.5.6, !=1.5.7',
+    'brotli',
     'pytest>=3'
 ]
 
@@ -105,6 +106,7 @@ def run_tests(self):
         'security': ['pyOpenSSL >= 0.14', 'cryptography>=1.3.4'],
         'socks': ['PySocks>=1.5.6, !=1.5.7'],
         'socks:sys_platform == "win32" and python_version == "2.7"': ['win_inet_pton'],
+        'brotli': ['urllib3 >= 1.25.1', 'brotli'],
     },
     project_urls={
         'Documentation': 'https://requests.readthedocs.io',

tests/test_lowlevel.py

@@ -3,7 +3,9 @@
 import pytest
 import threading
 import requests
+import brotli
 
+from requests.utils import brotli_supported
 from tests.testserver.server import Server, consume_socket_content
 
 from .utils import override_environ
@@ -307,3 +309,38 @@ def response_handler(sock):
         assert r.url == 'http://{}:{}/final-url/#relevant-section'.format(host, port)
 
         close_server.set()
+
+@pytest.mark.skipif(not brotli_supported(), reason='run only if Brotli is supported')
+def test_brotli():
+    """Verify that if Requests supports Brotli, then a request to a server serving
+    content in Brotli will result with a response with the content correctly decompressed.
+    """
+
+    some_text = '<html><html><html>'
+    br_compressed_text = brotli.compress(some_text.encode('utf-8'))
+
+    assert len(some_text) != len(br_compressed_text)
+    assert some_text != br_compressed_text
+
+    def response_handler(sock):
+        consume_socket_content(sock, timeout=0.5)
+
+        sock.send(
+            b'HTTP/1.1 200 OK\r\n'
+            b'Content-Length: ' + bytes(len(br_compressed_text)) + b'\r\n'
+            b'Content-Encoding: br\r\n'
+            b'\r\n' +
+            br_compressed_text
+        )
+
+    close_server = threading.Event()
+    server = Server(response_handler, wait_to_close_event=close_server)
+
+    with server as (host, port):
+        url = 'http://{}:{}/'.format(host, port)
+        r = requests.get(url)
+
+        assert len(r.text) == len(some_text)
+        assert r.text == some_text
+
+        close_server.set()

tests/test_requests.py

@@ -31,6 +31,7 @@
 from requests.models import urlencode
 from requests.hooks import default_hooks
 from requests.compat import MutableMapping
+from requests.utils import brotli_supported
 
 from .compat import StringIO, u
 from .utils import override_environ
@@ -2527,3 +2528,23 @@ def test_parameters_for_nonstandard_schemes(self, input, params, expected):
         r = requests.Request('GET', url=input, params=params)
         p = r.prepare()
         assert p.url == expected
+
+
+    @pytest.mark.skipif(not brotli_supported(), reason='run only if Brotli is supported')
+    def test_brotli_default_accept_encoding(self, httpbin):
+        """Verify that Requests with Brotli support by default sends Accept-Encoding with 'br',
+        as the first accepted encoding as the most effecient
+        """
+        r = requests.request('GET', httpbin('get'))
+
+        assert 'br' in r.request.headers['Accept-Encoding']
+        assert r.request.headers['Accept-Encoding'].startswith('br')
+
+
+    @pytest.mark.skipif(brotli_supported(), reason='run only if Brotli is NOT supported')
+    def test_no_brotli_default_accept_encoding(self, httpbin):
+        """Verify that Requests without Brotli support by default sends Accept-Encoding without 'br'
+        """
+        r = requests.request('GET', httpbin('get'))
+
+        assert 'br' not in r.request.headers['Accept-Encoding']