From 2fa3051b90e76b11fc572696ed7f0945a8e4c9ef Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 8 Jun 2023 17:04:22 +0200 Subject: [PATCH] Report gRPC connnection errors to the caller By default `grpc.DialContext()` is non-blocking so any connection issue will not be surfaced to the user. This change makes it blocking and configures the gRPC dialer to report the underlying error if any happens. Signed-off-by: Simon Pasquier --- CHANGELOG.md | 1 + pkg/query/endpointset.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13da39913f..468a36bbca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6342](https://github.com/thanos-io/thanos/pull/6342) Cache/Redis: Upgrade `rueidis` to v1.0.2 to to improve error handling while shrinking a redis cluster. - [#6325](https://github.com/thanos-io/thanos/pull/6325) Store: return gRPC resource exhausted error for byte limiter. - [#6399](https://github.com/thanos-io/thanos/pull/6399) *: Fix double-counting bug in http_request_duration metric +- [#6428](https://github.com/thanos-io/thanos/pull/6428) Report gRPC connnection errors in the logs. ### Changed - [#6168](https://github.com/thanos-io/thanos/pull/6168) Receiver: Make ketama hashring fail early when configured with number of nodes lower than the replication factor. diff --git a/pkg/query/endpointset.go b/pkg/query/endpointset.go index ad963edd3f..a92bcf39b6 100644 --- a/pkg/query/endpointset.go +++ b/pkg/query/endpointset.go @@ -636,6 +636,11 @@ type endpointRef struct { // The call to newEndpointRef will return an error if establishing the channel fails. func (e *EndpointSet) newEndpointRef(ctx context.Context, spec *GRPCEndpointSpec) (*endpointRef, error) { dialOpts := append(e.dialOpts, spec.dialOpts...) + // By default DialContext is non-blocking which means that any connection + // failure won't be reported/logged. Instead block until the connection is + // successfully established and return the details of the connection error + // if any. + dialOpts = append(dialOpts, grpc.WithReturnConnectionError()) conn, err := grpc.DialContext(ctx, spec.Addr(), dialOpts...) if err != nil { return nil, errors.Wrap(err, "dialing connection")