Clear fewer headers on 1xx/204/304 responses

This function is called on more than just 304 responses; it’s
important to permit the Allow header on 204 responses.  Also, the
relevant RFCs have changed significantly.

Fixes tornadoweb#2726.

Signed-off-by: Anders Kaseorg <andersk@mit.edu>

tornado/test/httpclient_test.py

@@ -109,7 +109,7 @@ def get(self):
         self.set_status(304)
         self.set_header("Content-Length", 42)
 
-    def _clear_headers_for_304(self):
+    def _clear_representation_headers(self):
         # Tornado strips content-length from 304 responses, but here we
         # want to simulate servers that include the headers anyway.
         pass

tornado/test/web_test.py

@@ -1175,7 +1175,6 @@ def test_static_304_if_modified_since(self):
         )
         self.assertEqual(response2.code, 304)
         self.assertTrue("Content-Length" not in response2.headers)
-        self.assertTrue("Last-Modified" not in response2.headers)
 
     def test_static_304_if_none_match(self):
         response1 = self.get_and_head("/static/robots.txt")

tornado/web.py

@@ -1138,7 +1138,7 @@ def finish(self, chunk: Optional[Union[str, bytes, dict]] = None) -> "Future[Non
                 assert not self._write_buffer, (
                     "Cannot send body with %s" % self._status_code
                 )
-                self._clear_headers_for_304()
+                self._clear_representation_headers()
             elif "Content-Length" not in self._headers:
                 content_length = sum(len(part) for part in self._write_buffer)
                 self.set_header("Content-Length", content_length)
@@ -1803,21 +1803,13 @@ def render(*args, **kwargs) -> str:  # type: ignore
     def _ui_method(self, method: Callable[..., str]) -> Callable[..., str]:
         return lambda *args, **kwargs: method(self, *args, **kwargs)
 
-    def _clear_headers_for_304(self) -> None:
-        # 304 responses should not contain entity headers (defined in
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec7.html#sec7.1)
+    def _clear_representation_headers(self) -> None:
+        # 304 responses should not contain representation metadata
+        # headers (defined in
+        # https://tools.ietf.org/html/rfc7231#section-3.1)
         # not explicitly allowed by
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5
-        headers = [
-            "Allow",
-            "Content-Encoding",
-            "Content-Language",
-            "Content-Length",
-            "Content-MD5",
-            "Content-Range",
-            "Content-Type",
-            "Last-Modified",
-        ]
+        # https://tools.ietf.org/html/rfc7232#section-4.1
+        headers = ["Content-Encoding", "Content-Language", "Content-Type"]
         for h in headers:
             self.clear_header(h)