From 755a890e7a4097db548ae53400532a692298fdee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ma=C5=9Blanka?= Date: Wed, 9 Oct 2024 11:52:28 +0200 Subject: [PATCH] c/producer_state: keep producer inflight requests queue bounded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Idempotent producers are allowed to produce up to 5 at a time, the five requests outcome is cached in the producer for deduplication. Previously the `_inflight_requests` list might grow unbounded if state machine apply was lagging. This causes performance issues and unnecessary memory pressure. Since the max inflight requests count is limited by producer it is enough to keep only last 5 inflight requests in the `_inflight_requests` list. Signed-off-by: Michał Maślanka --- src/v/cluster/producer_state.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/v/cluster/producer_state.cc b/src/v/cluster/producer_state.cc index acc883c3dc2b3..5f3b7364e01b8 100644 --- a/src/v/cluster/producer_state.cc +++ b/src/v/cluster/producer_state.cc @@ -187,6 +187,17 @@ result requests::try_emplace( // All invariants satisfied, enqueue the request. _inflight_requests.emplace_back( ss::make_lw_shared(first, last, current, result_promise_t{})); + // if there are more than max cached requests inflight start dropping the + // oldest of them + while (_inflight_requests.size() > requests_cached_max + && _inflight_requests.front()->has_completed()) { + _inflight_requests.pop_front(); + // clear finished requests as the producer will not be interested in + // them anymore + if (!_finished_requests.empty()) { + _finished_requests.clear(); + } + } return _inflight_requests.back(); }