[3.5] bpo-39503: CVE-2020-8492: Fix AbstractBasicAuthHandler (GH-18284)

Lib/test/test_urllib2.py

@@ -1361,40 +1361,64 @@ def test_osx_proxy_bypass(self):
  bypass = {'exclude_simple': True, 'exceptions': []}
  self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass))
 
- def test_basic_auth(self, quote_char='"'):
- opener = OpenerDirector()
- password_manager = MockPasswordManager()
- auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
- realm = "ACME Widget Store"
- http_handler = MockHTTPHandler(
- 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' %
- (quote_char, realm, quote_char))
- opener.add_handler(auth_handler)
- opener.add_handler(http_handler)
- self._test_basic_auth(opener, auth_handler, "Authorization",
- realm, http_handler, password_manager,
- "http://acme.example.com/protected",
- "http://acme.example.com/protected",
- )
-
- def test_basic_auth_with_single_quoted_realm(self):
- self.test_basic_auth(quote_char="'")
-
- def test_basic_auth_with_unquoted_realm(self):
- opener = OpenerDirector()
- password_manager = MockPasswordManager()
- auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
- realm = "ACME Widget Store"
- http_handler = MockHTTPHandler(
- 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm)
- opener.add_handler(auth_handler)
- opener.add_handler(http_handler)
- with self.assertWarns(UserWarning):
+ def check_basic_auth(self, headers, realm):
+ with self.subTest(realm=realm, headers=headers):
+ opener = OpenerDirector()
+ password_manager = MockPasswordManager()
+ auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
+ body = '\r\n'.join(headers) + '\r\n\r\n'
+ http_handler = MockHTTPHandler(401, body)
+ opener.add_handler(auth_handler)
+ opener.add_handler(http_handler)
  self._test_basic_auth(opener, auth_handler, "Authorization",
- realm, http_handler, password_manager,
- "http://acme.example.com/protected",
- "http://acme.example.com/protected",
- )
+ realm, http_handler, password_manager,
+ "http://acme.example.com/protected",
+ "http://acme.example.com/protected")
+
+ def test_basic_auth(self):
+ realm = "realm2@example.com"
+ realm2 = "realm2@example.com"
+ basic = 'Basic realm="{}"'.format(realm)
+ basic2 = 'Basic realm="{}"'.format(realm2)
+ other_no_realm = 'Otherscheme xxx'
+ digest = ('Digest realm="{}", '
+ 'qop="auth, auth-int", '
+ 'nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", '
+ 'opaque="5ccc069c403ebaf9f0171e9517f40e41"').format(realm2)
+ for realm_str in (
+ # test "quote" and 'quote'
+ 'Basic realm="{}"'.format(realm),
+ "Basic realm='{}'".format(realm),
+
+ # charset is ignored
+ 'Basic realm="{}", charset="UTF-8"'.format(realm),
+
+ # Multiple challenges per header
+ '{}, {}'.format(basic, basic2),
+ '{}, {}'.format(basic, other_no_realm),
+ '{}, {}'.format(other_no_realm, basic),
+ '{}, {}'.format(basic, digest),
+ '{}, {}'.format(digest, basic),
+ ):
+ headers = ['WWW-Authenticate: {}'.format(realm_str)]
+ self.check_basic_auth(headers, realm)
+
+ # no quote: expect a warning
+ with support.check_warnings(("Basic Auth Realm was unquoted",
+ UserWarning)):
+ headers = ['WWW-Authenticate: Basic realm={}'.format(realm)]
+ self.check_basic_auth(headers, realm)
+
+ # Multiple headers: one challenge per header.
+ # Use the first Basic realm.
+ for challenges in (
+ [basic, basic2],
+ [basic, digest],
+ [digest, basic],
+ ):
+ headers = ['WWW-Authenticate: {}'.format(challenge)
+ for challenge in challenges]
+ self.check_basic_auth(headers, realm)
 
  def test_proxy_basic_auth(self):
  opener = OpenerDirector()

Lib/urllib/request.py

@@ -885,8 +885,15 @@ class AbstractBasicAuthHandler:
 
  # allow for double- and single-quoted realm values
  # (single quotes are a violation of the RFC, but appear in the wild)
- rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
- 'realm=(["\']?)([^"\']*)\\2', re.I)
+ rx = re.compile('(?:^|,)' # start of the string or ','
+ '[ \t]*' # optional whitespaces
+ '([^ \t]+)' # scheme like "Basic"
+ '[ \t]+' # mandatory whitespaces
+ # realm=xxx
+ # realm='xxx'
+ # realm="xxx"
+ 'realm=(["\']?)([^"\']*)\\2',
+ re.I)
 
  # XXX could pre-emptively send auth info already accepted (RFC 2617,
  # end of section 2, and section 1.2 immediately after "credentials"
@@ -898,27 +905,51 @@ def __init__(self, password_mgr=None):
  self.passwd = password_mgr
  self.add_password = self.passwd.add_password
 
+ def _parse_realm(self, header):
+ # parse WWW-Authenticate header: accept multiple challenges per header
+ found_challenge = False
+ for mo in AbstractBasicAuthHandler.rx.finditer(header):
+ scheme, quote, realm = mo.groups()
+ if quote not in ['"', "'"]:
+ warnings.warn("Basic Auth Realm was unquoted",
+ UserWarning, 3)
+
+ yield (scheme, realm)
+
+ found_challenge = True
+
+ if not found_challenge:
+ if header:
+ scheme = header.split()[0]
+ else:
+ scheme = ''
+ yield (scheme, None)
+
  def http_error_auth_reqed(self, authreq, host, req, headers):
  # host may be an authority (without userinfo) or a URL with an
  # authority
- # XXX could be multiple headers
- authreq = headers.get(authreq, None)
+ headers = headers.get_all(authreq)
+ if not headers:
+ # no header found
+ return
 
- if authreq:
- scheme = authreq.split()[0]
- if scheme.lower() != 'basic':
- raise ValueError("AbstractBasicAuthHandler does not"
- " support the following scheme: '%s'" %
- scheme)
- else:
- mo = AbstractBasicAuthHandler.rx.search(authreq)
- if mo:
- scheme, quote, realm = mo.groups()
- if quote not in ['"',"'"]:
- warnings.warn("Basic Auth Realm was unquoted",
- UserWarning, 2)
- if scheme.lower() == 'basic':
- return self.retry_http_basic_auth(host, req, realm)
+ unsupported = None
+ for header in headers:
+ for scheme, realm in self._parse_realm(header):
+ if scheme.lower() != 'basic':
+ unsupported = scheme
+ continue
+
+ if realm is not None:
+ # Use the first matching Basic challenge.
+ # Ignore following challenges even if they use the Basic
+ # scheme.
+ return self.retry_http_basic_auth(host, req, realm)
+
+ if unsupported is not None:
+ raise ValueError("AbstractBasicAuthHandler does not "
+ "support the following scheme: %r"
+ % (scheme,))
 
  def retry_http_basic_auth(self, host, req, realm):
  user, pw = self.passwd.find_user_password(realm, host)

Misc/NEWS.d/next/Library/2020-03-25-16-02-16.bpo-39503.YmMbYn.rst

@@ -0,0 +1,3 @@
+:class:`~urllib.request.AbstractBasicAuthHandler` of :mod:`urllib.request`
+now parses all WWW-Authenticate HTTP headers and accepts multiple challenges
+per header: use the realm of the first Basic challenge.

Misc/NEWS.d/next/Security/2020-01-30-16-15-29.bpo-39503.B299Yq.rst

@@ -0,0 +1,5 @@
+CVE-2020-8492: The :class:`~urllib.request.AbstractBasicAuthHandler` class of the
+:mod:`urllib.request` module uses an inefficient regular expression which can
+be exploited by an attacker to cause a denial of service. Fix the regex to
+prevent the catastrophic backtracking. Vulnerability reported by Ben Caller
+and Matt Schwager.