From 7a8a4d0c6cbe8773147a2ed1445fdbb5ce751ec7 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Thu, 22 Jun 2023 18:42:24 -0700 Subject: [PATCH 1/2] Do not reuse remote write requests in case of error Signed-off-by: Alan Protasio --- pkg/distributor/distributor.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 7bb113e3ed0..bdd10333764 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -874,7 +874,12 @@ func (d *Distributor) send(ctx context.Context, ingester ring.InstanceDesc, time req.Source = source _, err = c.PushPreAlloc(ctx, req) - cortexpb.ReuseWriteRequest(req) + + // We should not reuse the req in case of errors: + // See: https://github.com/grpc/grpc-go/issues/6355 + if err == nil { + cortexpb.ReuseWriteRequest(req) + } if len(metadata) > 0 { d.ingesterAppends.WithLabelValues(ingester.Addr, typeMetadata).Inc() From 88eee30e955102623932fbc2c982864e09b83baf Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Thu, 22 Jun 2023 18:46:33 -0700 Subject: [PATCH 2/2] Changelog Signed-off-by: Alan Protasio --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6360bb8a7c2..9243f41acaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ * [BUGFIX] Compactor: Partial block with only visit marker should be deleted even there is no deletion marker. #5342 * [BUGFIX] KV: Etcd calls will no longer block indefinitely and will now time out after the DialTimeout period. #5392 * [BUGFIX] Ring: Allow RF greater than number of zones to select more than one instance per zone #5411 +* [BUGFIX] Distributor: Fix potential data corruption in cases of timeout between distributors and ingesters. #5422 ## 1.15.1 2023-04-26