From ab413d1d16ecf44e965471c0def56dde4a541032 Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Fri, 17 Jul 2020 02:18:08 +0530 Subject: [PATCH] fix(ServeTask): Return error if server is not ready (#6019) (#6020) The grpcWorker.ServeTask function is called when a request is received over the network (sent by a peer). This function doesn't consider the health of the node before starting the processing of the request. If we process a request before the node is ready, we might end up seeing crashes similar to https://dgraph.atlassian.net/browse/DGRAPH-1934 Fixes DGRAPH-1934 This bug was seen when running jepsen bank test with kill-alpha and kill-zero nemesis. (cherry picked from commit 9293c97cf8641a6ecbdbe316a312d183065d44c7) --- worker/task.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/worker/task.go b/worker/task.go index 36397459f8c..337775a3106 100644 --- a/worker/task.go +++ b/worker/task.go @@ -1854,6 +1854,12 @@ func (w *grpcWorker) ServeTask(ctx context.Context, q *pb.Query) (*pb.Result, er return nil, ctx.Err() } + // It could be possible that the server isn't ready but a peer sends a + // request. In that case we should check for the health here. + if err := x.HealthCheck(); err != nil { + return nil, err + } + gid, err := groups().BelongsToReadOnly(q.Attr, q.ReadTs) switch { case err != nil: