Skip to content

Save and propagate epoll network error #7809

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shinrich merged 2 commits into from
May 18, 2021
Merged

Save and propagate epoll network error #7809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9576d8d
Save and propagate epoll network error
shinrich May 11, 2021
d78df2f
adjust method name
shinrich May 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions iocore/net/NetEvent.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ class NetEvent
// Close when EventIO close;
virtual int close() = 0;

bool has_error() const;
void set_error_from_socket();

// get fd
virtual int get_fd() = 0;
virtual Ptr<ProxyMutex> &get_mutex() = 0;
Expand All @@ -65,6 +68,7 @@ class NetEvent
NetState write{};

int closed = 0;
int error = 0;
NetHandler *nh = nullptr;

ink_hrtime inactivity_timeout_in = 0;
Expand Down Expand Up @@ -94,3 +98,16 @@ class NetEvent
} f;
};
};

inline bool
NetEvent::has_error() const
{
return error != 0;
}

inline void
NetEvent::set_error_from_socket()
{
socklen_t errlen = sizeof(error);
getsockopt(this->get_fd(), SOL_SOCKET, SO_ERROR, (void *)&error, &errlen);
}
19 changes: 8 additions & 11 deletions iocore/net/UnixNet.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -507,27 +507,24 @@ NetHandler::waitForActivity(ink_hrtime timeout)
if (cop_list.in(ne)) {
cop_list.remove(ne);
}
if (get_ev_events(pd, x) & (EVENTIO_READ | EVENTIO_ERROR)) {
int flags = get_ev_events(pd, x);
Copy link
Member

@SolidWallOfCode SolidWallOfCode May 12, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to get only error bits and not read or write ready bits? It's reasonable to presume there's no epoll result unless checking for one of those conditions, so it's possible the checked for condition is always present.

Copy link
Member Author

@shinrich shinrich May 13, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is a ink_release_assert later in this function for that case. I have yet to see it go off.

if (flags & (EVENTIO_ERROR)) {
ne->set_error_from_socket();
}
if (flags & (EVENTIO_READ)) {
ne->read.triggered = 1;
if (!read_ready_list.in(ne)) {
read_ready_list.enqueue(ne);
} else if (get_ev_events(pd, x) & EVENTIO_ERROR) {
// check for unhandled epoll events that should be handled
Debug("iocore_net_main", "Unhandled epoll event on read: 0x%04x read.enabled=%d closed=%d read.netready_queue=%d",
get_ev_events(pd, x), ne->read.enabled, ne->closed, read_ready_list.in(ne));
}
}
if (get_ev_events(pd, x) & (EVENTIO_WRITE | EVENTIO_ERROR)) {
if (flags & (EVENTIO_WRITE)) {
ne->write.triggered = 1;
if (!write_ready_list.in(ne)) {
write_ready_list.enqueue(ne);
} else if (get_ev_events(pd, x) & EVENTIO_ERROR) {
// check for unhandled epoll events that should be handled
Debug("iocore_net_main", "Unhandled epoll event on write: 0x%04x write.enabled=%d closed=%d write.netready_queue=%d",
get_ev_events(pd, x), ne->write.enabled, ne->closed, write_ready_list.in(ne));
}
} else if (!(get_ev_events(pd, x) & EVENTIO_READ)) {
} else if (!(flags & (EVENTIO_READ))) {
Debug("iocore_net_main", "Unhandled epoll event: 0x%04x", get_ev_events(pd, x));
ink_release_assert(false);
Copy link
Contributor

@jrushford jrushford Jul 13, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@shinrich we're sending production traffic through 9.1.x and seeing it crash here with the assertion, line 527.

}
} else if (epd->type == EVENTIO_DNS_CONNECTION) {
if (epd->data.dnscon != nullptr) {
Expand Down
12 changes: 12 additions & 0 deletions iocore/net/UnixNetVConnection.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,12 @@ read_from_net(NetHandler *nh, UnixNetVConnection *vc, EThread *thread)
return;
}

if (vc->has_error()) {
vc->lerrno = vc->error;
vc->readSignalAndUpdate(VC_EVENT_ERROR);
return;
}

// It is possible that the closed flag got set from HttpSessionManager in the
// global session pool case. If so, the closed flag should be stable once we get the
// s->vio.mutex (the global session pool mutex).
Expand Down Expand Up @@ -368,6 +374,12 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread)
return;
}

if (vc->has_error()) {
vc->lerrno = vc->error;
write_signal_and_update(VC_EVENT_ERROR, vc);
return;
}

// This function will always return true unless
// vc is an SSLNetVConnection.
if (!vc->getSSLHandShakeComplete()) {
Expand Down