TS-4796 Change UnixNetHandler to always bubble up epoll errors to the VConnection

Before if the vcon wasn't read or write enabled errors would be swallowed. This leads to a variety of issues where the socket errors aren't dealt with immediately.

Author: jacksontj

iocore/net/P_UnixNetState.h

@@ -48,6 +48,7 @@ class UnixNetVConnection;
 
 struct NetState {
   volatile int enabled;
+  volatile int error;
   VIO vio;
   Link<UnixNetVConnection> ready_link;
   SLink<UnixNetVConnection> enable_link;

iocore/net/P_UnixNetVConnection.h

@@ -215,6 +215,7 @@ class UnixNetVConnection : public NetVConnection
   virtual int64_t load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs);
   void readDisable(NetHandler *nh);
   void readSignalError(NetHandler *nh, int err);
+  void writeSignalError(NetHandler *nh, int err);
   int readSignalDone(int event, NetHandler *nh);
   int readSignalAndUpdate(int event);
   void readReschedule(NetHandler *nh);

iocore/net/UnixNet.cc

@@ -480,6 +480,9 @@ NetHandler::mainNetEvent(int event, Event *e)
       vc = epd->data.vc;
       if (get_ev_events(pd, x) & (EVENTIO_READ | EVENTIO_ERROR)) {
         vc->read.triggered = 1;
+        if (get_ev_events(pd, x) & EVENTIO_ERROR) {
+          vc->read.error = 1;
+        }
         if (!read_ready_list.in(vc)) {
           read_ready_list.enqueue(vc);
         } else if (get_ev_events(pd, x) & EVENTIO_ERROR) {
@@ -491,6 +494,9 @@ NetHandler::mainNetEvent(int event, Event *e)
       vc = epd->data.vc;
       if (get_ev_events(pd, x) & (EVENTIO_WRITE | EVENTIO_ERROR)) {
         vc->write.triggered = 1;
+        if (get_ev_events(pd, x) & EVENTIO_ERROR) {
+          vc->write.error = 1;
+        }
         if (!write_ready_list.in(vc)) {
           write_ready_list.enqueue(vc);
         } else if (get_ev_events(pd, x) & EVENTIO_ERROR) {
@@ -525,7 +531,17 @@ NetHandler::mainNetEvent(int event, Event *e)
       close_UnixNetVConnection(vc, trigger_event->ethread);
     else if (vc->read.enabled && vc->read.triggered)
       vc->net_read_io(this, trigger_event->ethread);
-    else if (!vc->read.enabled) {
+    else if (vc->read.error) {
+      int err = 0, errlen = sizeof(int);
+      if (getsockopt(vc->con.fd, SOL_SOCKET, SO_ERROR, &err, (socklen_t *)&errlen) == -1) {
+        err = errno;
+      }
+      if (err != EAGAIN && err != EINTR) {
+        vc->readSignalError(this, err);
+      } else {
+        vc->read.error = 0;
+      }
+    } else if (!vc->read.enabled) {
       read_ready_list.remove(vc);
 #if defined(solaris)
       if (vc->read.triggered && vc->write.enabled) {
@@ -542,7 +558,17 @@ NetHandler::mainNetEvent(int event, Event *e)
       close_UnixNetVConnection(vc, trigger_event->ethread);
     else if (vc->write.enabled && vc->write.triggered)
       write_to_net(this, vc, trigger_event->ethread);
-    else if (!vc->write.enabled) {
+    else if (vc->write.error) {
+      int err = 0, errlen = sizeof(int);
+      if (getsockopt(vc->con.fd, SOL_SOCKET, SO_ERROR, &err, (socklen_t *)&errlen) == -1) {
+        err = errno;
+      }
+      if (err != EAGAIN && err != EINTR) {
+        vc->writeSignalError(this, err);
+      } else {
+        vc->write.error = 0;
+      }
+    } else if (!vc->write.enabled) {
       write_ready_list.remove(vc);
 #if defined(solaris)
       if (vc->write.triggered && vc->read.enabled) {

iocore/net/UnixNetVConnection.cc

@@ -1024,6 +1024,12 @@ UnixNetVConnection::readSignalError(NetHandler *nh, int err)
   read_signal_error(nh, this, err);
 }
 
+void
+UnixNetVConnection::writeSignalError(NetHandler *nh, int err)
+{
+  write_signal_error(nh, this, err);
+}
+
 int
 UnixNetVConnection::readSignalDone(int event, NetHandler *nh)
 {