From e213d7404a0384c68cad482b687e241c2ae3f711 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 2 Apr 2020 02:52:20 +0200 Subject: [PATCH] bpo-39503: CVE-2020-8492: Fix AbstractBasicAuthHandler (GH-18284) The AbstractBasicAuthHandler class of the urllib.request module uses an inefficient regular expression which can be exploited by an attacker to cause a denial of service. Fix the regex to prevent the catastrophic backtracking. Vulnerability reported by Ben Caller and Matt Schwager. AbstractBasicAuthHandler of urllib.request now parses all WWW-Authenticate HTTP headers and accepts multiple challenges per header: use the realm of the first Basic challenge. Co-Authored-By: Serhiy Storchaka (cherry picked from commit 0b297d4ff1c0e4480ad33acae793fbaf4bf015b4) --- Lib/test/test_urllib2.py | 90 ++++++++++++------- Lib/urllib/request.py | 69 ++++++++++---- .../2020-03-25-16-02-16.bpo-39503.YmMbYn.rst | 3 + .../2020-01-30-16-15-29.bpo-39503.B299Yq.rst | 5 ++ 4 files changed, 115 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-03-25-16-02-16.bpo-39503.YmMbYn.rst create mode 100644 Misc/NEWS.d/next/Security/2020-01-30-16-15-29.bpo-39503.B299Yq.rst diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 3ed81ce5105119..8e5b68e68aa95b 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1361,40 +1361,64 @@ def test_osx_proxy_bypass(self): bypass = {'exclude_simple': True, 'exceptions': []} self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass)) - def test_basic_auth(self, quote_char='"'): - opener = OpenerDirector() - password_manager = MockPasswordManager() - auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) - realm = "ACME Widget Store" - http_handler = MockHTTPHandler( - 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' % - (quote_char, realm, quote_char)) - opener.add_handler(auth_handler) - opener.add_handler(http_handler) - self._test_basic_auth(opener, auth_handler, "Authorization", - realm, http_handler, password_manager, - "http://acme.example.com/protected", - "http://acme.example.com/protected", - ) - - def test_basic_auth_with_single_quoted_realm(self): - self.test_basic_auth(quote_char="'") - - def test_basic_auth_with_unquoted_realm(self): - opener = OpenerDirector() - password_manager = MockPasswordManager() - auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) - realm = "ACME Widget Store" - http_handler = MockHTTPHandler( - 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm) - opener.add_handler(auth_handler) - opener.add_handler(http_handler) - with self.assertWarns(UserWarning): + def check_basic_auth(self, headers, realm): + with self.subTest(realm=realm, headers=headers): + opener = OpenerDirector() + password_manager = MockPasswordManager() + auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) + body = '\r\n'.join(headers) + '\r\n\r\n' + http_handler = MockHTTPHandler(401, body) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) self._test_basic_auth(opener, auth_handler, "Authorization", - realm, http_handler, password_manager, - "http://acme.example.com/protected", - "http://acme.example.com/protected", - ) + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected") + + def test_basic_auth(self): + realm = "realm2@example.com" + realm2 = "realm2@example.com" + basic = 'Basic realm="{}"'.format(realm) + basic2 = 'Basic realm="{}"'.format(realm2) + other_no_realm = 'Otherscheme xxx' + digest = ('Digest realm="{}", ' + 'qop="auth, auth-int", ' + 'nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", ' + 'opaque="5ccc069c403ebaf9f0171e9517f40e41"').format(realm2) + for realm_str in ( + # test "quote" and 'quote' + 'Basic realm="{}"'.format(realm), + "Basic realm='{}'".format(realm), + + # charset is ignored + 'Basic realm="{}", charset="UTF-8"'.format(realm), + + # Multiple challenges per header + '{}, {}'.format(basic, basic2), + '{}, {}'.format(basic, other_no_realm), + '{}, {}'.format(other_no_realm, basic), + '{}, {}'.format(basic, digest), + '{}, {}'.format(digest, basic), + ): + headers = ['WWW-Authenticate: {}'.format(realm_str)] + self.check_basic_auth(headers, realm) + + # no quote: expect a warning + with support.check_warnings(("Basic Auth Realm was unquoted", + UserWarning)): + headers = ['WWW-Authenticate: Basic realm={}'.format(realm)] + self.check_basic_auth(headers, realm) + + # Multiple headers: one challenge per header. + # Use the first Basic realm. + for challenges in ( + [basic, basic2], + [basic, digest], + [digest, basic], + ): + headers = ['WWW-Authenticate: {}'.format(challenge) + for challenge in challenges] + self.check_basic_auth(headers, realm) def test_proxy_basic_auth(self): opener = OpenerDirector() diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 3184ddab24a0b4..ae3fd0d10b11d3 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -885,8 +885,15 @@ class AbstractBasicAuthHandler: # allow for double- and single-quoted realm values # (single quotes are a violation of the RFC, but appear in the wild) - rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' - 'realm=(["\']?)([^"\']*)\\2', re.I) + rx = re.compile('(?:^|,)' # start of the string or ',' + '[ \t]*' # optional whitespaces + '([^ \t]+)' # scheme like "Basic" + '[ \t]+' # mandatory whitespaces + # realm=xxx + # realm='xxx' + # realm="xxx" + 'realm=(["\']?)([^"\']*)\\2', + re.I) # XXX could pre-emptively send auth info already accepted (RFC 2617, # end of section 2, and section 1.2 immediately after "credentials" @@ -898,27 +905,51 @@ def __init__(self, password_mgr=None): self.passwd = password_mgr self.add_password = self.passwd.add_password + def _parse_realm(self, header): + # parse WWW-Authenticate header: accept multiple challenges per header + found_challenge = False + for mo in AbstractBasicAuthHandler.rx.finditer(header): + scheme, quote, realm = mo.groups() + if quote not in ['"', "'"]: + warnings.warn("Basic Auth Realm was unquoted", + UserWarning, 3) + + yield (scheme, realm) + + found_challenge = True + + if not found_challenge: + if header: + scheme = header.split()[0] + else: + scheme = '' + yield (scheme, None) + def http_error_auth_reqed(self, authreq, host, req, headers): # host may be an authority (without userinfo) or a URL with an # authority - # XXX could be multiple headers - authreq = headers.get(authreq, None) + headers = headers.get_all(authreq) + if not headers: + # no header found + return - if authreq: - scheme = authreq.split()[0] - if scheme.lower() != 'basic': - raise ValueError("AbstractBasicAuthHandler does not" - " support the following scheme: '%s'" % - scheme) - else: - mo = AbstractBasicAuthHandler.rx.search(authreq) - if mo: - scheme, quote, realm = mo.groups() - if quote not in ['"',"'"]: - warnings.warn("Basic Auth Realm was unquoted", - UserWarning, 2) - if scheme.lower() == 'basic': - return self.retry_http_basic_auth(host, req, realm) + unsupported = None + for header in headers: + for scheme, realm in self._parse_realm(header): + if scheme.lower() != 'basic': + unsupported = scheme + continue + + if realm is not None: + # Use the first matching Basic challenge. + # Ignore following challenges even if they use the Basic + # scheme. + return self.retry_http_basic_auth(host, req, realm) + + if unsupported is not None: + raise ValueError("AbstractBasicAuthHandler does not " + "support the following scheme: %r" + % (scheme,)) def retry_http_basic_auth(self, host, req, realm): user, pw = self.passwd.find_user_password(realm, host) diff --git a/Misc/NEWS.d/next/Library/2020-03-25-16-02-16.bpo-39503.YmMbYn.rst b/Misc/NEWS.d/next/Library/2020-03-25-16-02-16.bpo-39503.YmMbYn.rst new file mode 100644 index 00000000000000..be80ce79d91edf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-03-25-16-02-16.bpo-39503.YmMbYn.rst @@ -0,0 +1,3 @@ +:class:`~urllib.request.AbstractBasicAuthHandler` of :mod:`urllib.request` +now parses all WWW-Authenticate HTTP headers and accepts multiple challenges +per header: use the realm of the first Basic challenge. diff --git a/Misc/NEWS.d/next/Security/2020-01-30-16-15-29.bpo-39503.B299Yq.rst b/Misc/NEWS.d/next/Security/2020-01-30-16-15-29.bpo-39503.B299Yq.rst new file mode 100644 index 00000000000000..9f2800581ca5ea --- /dev/null +++ b/Misc/NEWS.d/next/Security/2020-01-30-16-15-29.bpo-39503.B299Yq.rst @@ -0,0 +1,5 @@ +CVE-2020-8492: The :class:`~urllib.request.AbstractBasicAuthHandler` class of the +:mod:`urllib.request` module uses an inefficient regular expression which can +be exploited by an attacker to cause a denial of service. Fix the regex to +prevent the catastrophic backtracking. Vulnerability reported by Ben Caller +and Matt Schwager.