bug: Silence remaining nuisance alarms

Issue #1123

autopush/router/apnsrouter.py

@@ -1,5 +1,5 @@
 """APNS Router"""
-import ssl
+import socket
 import uuid
 from typing import Any  # noqa
 
@@ -157,22 +157,11 @@ def _route(self, notification, router_data):
                                        self._base_tags,
                                        application=rel_channel,
                                        reason="connection_error"))
-        except HTTP20Error:
+        except (HTTP20Error, socket.error):
             self.metrics.increment("notification.bridge.connection.error",
                                    tags=make_tags(self._base_tags,
                                                   application=rel_channel,
                                                   reason="http2_error"))
-        except ssl.SSLError as e:
-            # can only str match this (for autopush#1048)
-            if not (e.errno == ssl.SSL_ERROR_SSL and
-                    str(e).startswith("[SSL: BAD_WRITE_RETRY]")):
-                raise  # pragma: nocover
-            self.metrics.increment(
-                "notification.bridge.connection.error",
-                tags=make_tags(self._base_tags,
-                               application=rel_channel,
-                               reason="bad_write_retry_error")
-            )
         if not success:
             raise RouterException(
                 "Server error",

autopush/tests/test_router.py

@@ -5,6 +5,7 @@
 import json
 import decimal
 import requests
+import socket
 import ssl
 
 import pytest
@@ -164,6 +165,25 @@ def raiser(*args, **kwargs):
         assert ex.value.status_code == 502
         self.flushLoggedErrors()
 
+    @inlineCallbacks
+    def test_connection_fail_error(self):
+
+        def raiser(*args, **kwargs):
+            error = socket.error()
+            error.errno = socket.errno.EPIPE
+            raise error
+
+        self.router.apns['firefox'].connections[1].request = Mock(
+            side_effect=raiser)
+
+        with pytest.raises(RouterException) as ex:
+            yield self.router.route_notification(self.notif, self.router_data)
+
+        assert ex.value.response_body == "APNS returned an error processing " \
+                                         "request"
+        assert ex.value.status_code == 502
+        self.flushLoggedErrors()
+
     @inlineCallbacks
     def test_route_notification(self):
         result = yield self.router.route_notification(self.notif,

autopush/tests/test_web_base.py

@@ -193,6 +193,15 @@ def test_response_err(self):
         self.base._response_err(fail)
         self.status_mock.assert_called_with(500, reason=None)
 
+    def test_connection_err(self):
+        from twisted.internet.error import ConnectionDone
+        try:
+            raise ConnectionDone("Connection was closed cleanly.")
+        except Exception:
+            fail = Failure()
+        self.base._response_err(fail)
+        assert not self.status_mock.called
+
     def test_overload_err(self):
         try:
             raise ClientError(

autopush/tests/test_websocket.py

@@ -4,6 +4,7 @@
 import uuid
 from hashlib import sha256
 from collections import defaultdict
+from urllib3.exceptions import ConnectTimeoutError
 
 import twisted.internet.base
 from autobahn.twisted.util import sleep
@@ -1064,6 +1065,20 @@ def test_register_kill_others_fail(self):
         d.errback(ConnectError())
         return d
 
+    def test_connection_timeout_fail(self):
+        self._connect()
+
+        d = Deferred()
+        self.mock_agent.request.return_value = d
+        node_id = "http://otherhost"
+        uaid = "deadbeef000000000000000000000000"
+        self.proto.ps.uaid = uaid
+        connected = int(time.time())
+        res = dict(node_id=node_id, connected_at=connected, uaid=uaid)
+        self.proto._check_other_nodes((True, res))
+        d.errback(ConnectTimeoutError())
+        return d
+
     @inlineCallbacks
     def test_register_over_provisioning(self):
         self._connect()

autopush/web/base.py

@@ -238,7 +238,10 @@ def _response_err(self, fail):
         running.
 
         """
+        from twisted.internet.error import ConnectionDone
         fmt = fail.value.message or 'Exception'
+        if isinstance(fail.value, ConnectionDone):
+            return
         self.log.failure(format=fmt, failure=fail,
                          status_code=500, errno=999,
                          client_info=self._client_info)

autopush/websocket.py

@@ -52,6 +52,7 @@
     ItemNotFound
 )
 from botocore.exceptions import ClientError
+from botocore.vendored.requests.packages import urllib3
 from twisted.internet import reactor
 from twisted.internet.defer import (
     Deferred,
@@ -60,8 +61,8 @@
 )
 from twisted.internet.error import (
     ConnectError,
-    ConnectionClosed
-)
+    ConnectionClosed,
+    DNSLookupError)
 from twisted.internet.interfaces import IProducer
 from twisted.internet.threads import deferToThread
 from twisted.logger import Logger
@@ -375,7 +376,12 @@ def trap_cancel(self, fail):
         fail.trap(CancelledError)
 
     def trap_connection_err(self, fail):
-        fail.trap(ConnectError, ConnectionClosed, ResponseFailed)
+        fail.trap(ConnectError, ConnectionClosed, ResponseFailed,
+                  DNSLookupError)
+
+    def trap_boto3_err(self, fail):
+        # trap boto3 ConnectTimeoutError in retry
+        fail.trap(urllib3.exceptions.ConnectTimeoutError)
 
     def force_retry(self, func, *args, **kwargs):
         # type: (Callable[..., Any], *Any, **Any) -> Deferred
@@ -629,6 +635,7 @@ def _notify_node(self, result):
             url.encode("utf8"),
         ).addCallback(IgnoreBody.ignore)
         d.addErrback(self.trap_connection_err)
+        d.addErrback(self.trap_boto3_err)
         d.addErrback(self.log_failure, extra="Failed to notify node")
 
     def returnError(self, messageType, reason, statusCode, close=True,
@@ -837,6 +844,7 @@ def _check_other_nodes(self, result, url=DEFAULT_WS_ERR):
                 url = "%s/notif/%s/%s" % (node_id, self.ps.uaid, last_connect)
                 d = self.factory.agent.request("DELETE", url.encode("utf8"))
                 d.addErrback(self.trap_connection_err)
+                d.addErrback(self.trap_boto3_err)
                 d.addErrback(self.log_failure,
                              extra="Failed to delete old node")