From eee3515af7e2c4991741c11b9d20515ec7c912a4 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sun, 29 Sep 2024 17:05:30 +0900 Subject: [PATCH 01/30] Test --- .../KafkaMessageListenerContainer.java | 42 +++++++++++++++++++ .../kafka/listener/MessageListener.java | 7 +++- ...fkaBackoffAwareMessageListenerAdapter.java | 8 +++- .../MessagingMessageListenerAdapter.java | 17 ++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index bbc8ae8ccb..6a734e2892 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -16,6 +16,7 @@ package org.springframework.kafka.listener; +import java.lang.Thread.UncaughtExceptionHandler; import java.nio.ByteBuffer; import java.time.Duration; import java.util.AbstractMap.SimpleEntry; @@ -895,6 +896,11 @@ else if (listener instanceof MessageListener) { this.wantsFullRecords = false; this.pollThreadStateProcessor = setUpPollProcessor(false); this.observationEnabled = this.containerProperties.isObservationEnabled(); + + final BiConsumer, RuntimeException> asyncRetryCallback + = (cRecord, runtimeException) -> + this.invokeErrorHandlerBySingleRecord(cRecord, runtimeException); + this.listener.setAsyncRetryCallback(asyncRetryCallback); } else { throw new IllegalArgumentException("Listener must be one of 'MessageListener', " @@ -2827,6 +2833,42 @@ private void doInvokeOnMessage(final ConsumerRecord recordArg) { } } + private void invokeErrorHandlerBySingleRecord(final ConsumerRecord cRecord, RuntimeException rte) { + if (this.commonErrorHandler.seeksAfterHandling() || rte instanceof CommitFailedException) { + try { + if (this.producer == null) { + processCommits(); + } + } + catch (Exception ex) { // NO SONAR + this.logger.error(ex, "Failed to commit before handling error"); + } + List> records = new ArrayList<>(); + records.add(cRecord); + this.commonErrorHandler.handleRemaining(rte, records, this.consumer, + KafkaMessageListenerContainer.this.thisOrParentContainer); + } + else { + boolean handled = false; + try { + handled = this.commonErrorHandler.handleOne(rte, cRecord, this.consumer, + KafkaMessageListenerContainer.this.thisOrParentContainer); + } + catch (Exception ex) { + this.logger.error(ex, "ErrorHandler threw unexpected exception"); + } + Map>> records = new LinkedHashMap<>(); + if (!handled) { + records.computeIfAbsent(new TopicPartition(cRecord.topic(), cRecord.partition()), + tp -> new ArrayList<>()).add(cRecord); + } + if (!records.isEmpty()) { + this.remainingRecords = new ConsumerRecords<>(records); + this.pauseForPending = true; + } + } + } + private void invokeErrorHandler(final ConsumerRecord cRecord, Iterator> iterator, RuntimeException rte) { diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java index ab1c5a97b9..ccce3541e7 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2019 the original author or authors. + * Copyright 2015-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package org.springframework.kafka.listener; +import java.util.function.BiConsumer; + import org.apache.kafka.clients.consumer.ConsumerRecord; /** @@ -30,4 +32,7 @@ @FunctionalInterface public interface MessageListener extends GenericMessageListener> { + default void setAsyncRetryCallback(BiConsumer, RuntimeException> asyncRetryCallback) { + // + } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java index d4eb63e903..efd4e63f28 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 the original author or authors. + * Copyright 2018-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import java.time.Clock; import java.time.Instant; import java.util.Optional; +import java.util.function.BiConsumer; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -143,4 +144,9 @@ public void onMessage(ConsumerRecord data, Acknowledgment acknowledgment) public void onMessage(ConsumerRecord data, Consumer consumer) { onMessage(data, null, consumer); } + + @Override + public void setAsyncRetryCallback(BiConsumer, RuntimeException> asyncRetryCallback) { + this.delegate.setAsyncRetryCallback(asyncRetryCallback); + } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index 9ae1567e67..73728113f1 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; import java.util.stream.Collectors; import org.apache.commons.logging.LogFactory; @@ -46,6 +47,7 @@ import org.springframework.expression.spel.support.StandardEvaluationContext; import org.springframework.expression.spel.support.StandardTypeConverter; import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.listener.AsyncRetryableException; import org.springframework.kafka.listener.ConsumerSeekAware; import org.springframework.kafka.listener.KafkaListenerErrorHandler; import org.springframework.kafka.listener.ListenerExecutionFailedException; @@ -152,6 +154,8 @@ public abstract class MessagingMessageListenerAdapter implements ConsumerS private String correlationHeaderName = KafkaHeaders.CORRELATION_ID; + private BiConsumer, RuntimeException> asyncRetryCallback; + /** * Create an instance with the provided bean and method. * @param bean the bean. @@ -664,6 +668,15 @@ protected void acknowledge(@Nullable Acknowledgment acknowledgment) { protected void asyncFailure(Object request, @Nullable Acknowledgment acknowledgment, Consumer consumer, Throwable t, Message source) { + if (t.getCause() instanceof AsyncRetryableException) { + if (request instanceof ConsumerRecord) { + ConsumerRecord record = (ConsumerRecord) request; + AsyncRetryableException ex = (AsyncRetryableException) t.getCause(); + asyncRetryCallback.accept(record, ex); + return; + } + } + try { handleException(request, acknowledgment, consumer, source, new ListenerExecutionFailedException(createMessagingErrorMessage( @@ -887,4 +900,8 @@ public void acknowledge() { } + public void setAsyncRetryCallback(BiConsumer, RuntimeException> asyncRetryCallback) { + this.asyncRetryCallback = asyncRetryCallback; + } + } From fdc59f065f19d9db6ede26de62a3453a59ce4f24 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Mon, 30 Sep 2024 21:37:40 +0900 Subject: [PATCH 02/30] DRAFT --- .../KafkaMessageListenerContainer.java | 11 +- ...fkaBackoffAwareMessageListenerAdapter.java | 11 + .../MessagingMessageListenerAdapter.java | 30 +- ...cRetryTopicClassLevelIntegrationTests.java | 1096 +++++++++++++++++ 4 files changed, 1132 insertions(+), 16 deletions(-) create mode 100644 spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncRetryTopicClassLevelIntegrationTests.java diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index 6a734e2892..7bb04c4b21 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -74,6 +74,7 @@ import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; +import org.springframework.aop.support.AopUtils; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.ObjectProvider; import org.springframework.context.ApplicationContext; @@ -897,10 +898,12 @@ else if (listener instanceof MessageListener) { this.pollThreadStateProcessor = setUpPollProcessor(false); this.observationEnabled = this.containerProperties.isObservationEnabled(); - final BiConsumer, RuntimeException> asyncRetryCallback - = (cRecord, runtimeException) -> - this.invokeErrorHandlerBySingleRecord(cRecord, runtimeException); - this.listener.setAsyncRetryCallback(asyncRetryCallback); + if (!AopUtils.isAopProxy(listener)) { + final BiConsumer, RuntimeException> asyncRetryCallback + = (cRecord, runtimeException) -> + this.invokeErrorHandlerBySingleRecord(cRecord, runtimeException); + this.listener.setAsyncRetryCallback(asyncRetryCallback); + } } else { throw new IllegalArgumentException("Listener must be one of 'MessageListener', " diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java index efd4e63f28..74ea973dab 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java @@ -94,6 +94,17 @@ public void onMessage(ConsumerRecord consumerRecord, @Nullable Acknowledgm maybeGetBackoffTimestamp(consumerRecord) .ifPresent(nextExecutionTimestamp -> this.kafkaConsumerBackoffManager .backOffIfNecessary(createContext(consumerRecord, nextExecutionTimestamp, consumer))); + // ConsumerRecord(topic = myRetryFutureTopic5, partition = 1, leaderEpoch = 0, offset = 0, CreateTime = 1727697786519, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [], isReadOnly = false), key = 0, value = Testing topic 5 - 1) + // ConsumerRecord(topic = myRetryFutureTopic5-listener2-0, partition = 1, leaderEpoch = 0, offset = 0, CreateTime = 1727697787141, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 83, 116, 97, 103, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 56, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 119, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 50, 51, 50, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 56, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 105, 110, 118, 111, 107, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 51, 57, 50, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 56, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 53, 48, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 105, 110, 118, 111, 107, 101, 68, 101, 108, 101, 103, 97, 116, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 49, 49, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 49, 48, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 53, 48, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 56, 50, 49, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 57, 57, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 100, 111, 73, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 36, 53, 52, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 49, 55, 41, 10, 9, 97, 116, 32, 105, 111, 46, 109, 105, 99, 114, 111, 109, 101, 116, 101, 114, 46, 111, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 79, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 111, 98, 115, 101, 114, 118, 101, 40, 79, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 106, 97, 118, 97, 58, 53, 54, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 49, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 87, 105, 116, 104, 82, 101, 99, 111, 114, 100, 115, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 53, 53, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 52, 52, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 48, 57, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 73, 102, 72, 97, 118, 101, 82, 101, 99, 111, 114, 100, 115, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 52, 55, 51, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 112, 111, 108, 108, 65, 110, 100, 73, 110, 118, 111, 107, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 52, 51, 56, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 114, 117, 110, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 51, 48, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 82, 117, 110, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 56, 48, 52, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 56, 51, 51, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 53, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10]), RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 1]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -105]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21])], isReadOnly = false), key = 0, value = Testing topic 5 - 1) + // ConsumerRecord(topic = myRetryFutureTopic5-listener2-1, partition = 1, leaderEpoch = 0, offset = 0, CreateTime = 1727697788166, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 1]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -105]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 48]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 48, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 53, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -19])], isReadOnly = false), key = 0, value = Testing topic 5 - 1) + // ConsumerRecord(topic = myRetryFutureTopic5-listener2-2, partition = 1, leaderEpoch = 0, offset = 0, CreateTime = 1727697789173, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 1]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -105]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -19]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 49]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 49, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 53, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 4]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -35])], isReadOnly = false), key = 0, value = Testing topic 5 - 1) + // ConsumerRecord(topic = myRetryFutureTopic5-listener2-dlt, partition = 1, leaderEpoch = 0, offset = 0, CreateTime = 1727697790187, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 1]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -105]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -19]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 4]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -35]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 50]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 50, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 53, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 5]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -22])], isReadOnly = false), key = 0, value = Testing topic 5 - 1) + + // ConsumerRecord(topic = myRetryFutureTopic5, partition = 0, leaderEpoch = 0, offset = 0, CreateTime = 1727697786516, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [], isReadOnly = false), key = 0, value = Testing topic 5 - 0) + // ConsumerRecord(topic = myRetryFutureTopic5-listener1-0, partition = 0, leaderEpoch = 0, offset = 0, CreateTime = 1727697787141, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 83, 116, 97, 103, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 56, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 119, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 50, 51, 50, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 56, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 105, 110, 118, 111, 107, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 51, 57, 50, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 56, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 53, 48, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 105, 110, 118, 111, 107, 101, 68, 101, 108, 101, 103, 97, 116, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 49, 49, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 49, 48, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 53, 48, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 56, 50, 49, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 57, 57, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 100, 111, 73, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 36, 53, 52, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 49, 55, 41, 10, 9, 97, 116, 32, 105, 111, 46, 109, 105, 99, 114, 111, 109, 101, 116, 101, 114, 46, 111, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 79, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 111, 98, 115, 101, 114, 118, 101, 40, 79, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 106, 97, 118, 97, 58, 53, 54, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 49, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 87, 105, 116, 104, 82, 101, 99, 111, 114, 100, 115, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 53, 53, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 52, 52, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 48, 57, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 73, 102, 72, 97, 118, 101, 82, 101, 99, 111, 114, 100, 115, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 52, 55, 51, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 112, 111, 108, 108, 65, 110, 100, 73, 110, 118, 111, 107, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 52, 51, 56, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 114, 117, 110, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 51, 48, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 82, 117, 110, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 56, 48, 52, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 56, 51, 51, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 50, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10]), RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 0]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -108]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21])], isReadOnly = false), key = 0, value = Testing topic 5 - 0) + // ConsumerRecord(topic = myRetryFutureTopic5-listener1-1, partition = 0, leaderEpoch = 0, offset = 0, CreateTime = 1727697788165, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 0]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -108]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 48]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 48, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 50, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -20])], isReadOnly = false), key = 0, value = Testing topic 5 - 0) + // ConsumerRecord(topic = myRetryFutureTopic5-listener1-2, partition = 0, leaderEpoch = 0, offset = 0, CreateTime = 1727697789173, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 0]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -108]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -20]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 49]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 49, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 50, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 4]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -35])], isReadOnly = false), key = 0, value = Testing topic 5 - 0) + // ConsumerRecord(topic = myRetryFutureTopic5-listener1-dlt, partition = 0, leaderEpoch = 0, offset = 0, CreateTime = 1727697790187, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 0]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -108]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -20]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 4]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -35]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 50]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 50, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 50, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 5]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -22])], isReadOnly = false), key = 0, value = Testing topic 5 - 0) try { invokeDelegateOnMessage(consumerRecord, acknowledgment, consumer); } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index 73728113f1..4091a3c271 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.function.BiConsumer; import java.util.stream.Collectors; @@ -668,23 +669,28 @@ protected void acknowledge(@Nullable Acknowledgment acknowledgment) { protected void asyncFailure(Object request, @Nullable Acknowledgment acknowledgment, Consumer consumer, Throwable t, Message source) { - if (t.getCause() instanceof AsyncRetryableException) { - if (request instanceof ConsumerRecord) { - ConsumerRecord record = (ConsumerRecord) request; - AsyncRetryableException ex = (AsyncRetryableException) t.getCause(); - asyncRetryCallback.accept(record, ex); - return; - } - } - try { - handleException(request, acknowledgment, consumer, source, - new ListenerExecutionFailedException(createMessagingErrorMessage( - "Async Fail", source.getPayload()), t)); + if (t instanceof CompletionException) { + // For CompletableFuture Object + handleException(request, acknowledgment, consumer, source, + new ListenerExecutionFailedException(createMessagingErrorMessage( + "Async Fail", source.getPayload()), t.getCause())); + } + else { + // For Mono Object. + handleException(request, acknowledgment, consumer, source, + new ListenerExecutionFailedException(createMessagingErrorMessage( + "Async Fail", source.getPayload()), t)); + } } catch (Throwable ex) { this.logger.error(t, () -> "Future, Mono, or suspend function was completed with an exception for " + source); acknowledge(acknowledgment); + if (request instanceof ConsumerRecord && + ex instanceof RuntimeException) { + ConsumerRecord record = (ConsumerRecord) request; + asyncRetryCallback.accept(record, (RuntimeException) ex); + } } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncRetryTopicClassLevelIntegrationTests.java new file mode 100644 index 0000000000..97a629b3ec --- /dev/null +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncRetryTopicClassLevelIntegrationTests.java @@ -0,0 +1,1096 @@ +package org.springframework.kafka.retrytopic; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.annotation.DltHandler; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.annotation.KafkaHandler; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.annotation.PartitionOffset; +import org.springframework.kafka.annotation.RetryableTopic; +import org.springframework.kafka.annotation.TopicPartition; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.config.KafkaListenerEndpointRegistry; +import org.springframework.kafka.config.TopicBuilder; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaAdmin; +import org.springframework.kafka.core.KafkaAdmin.NewTopics; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.AsyncRetryableException; +import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; +import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.listener.ContainerProperties.AckMode; +import org.springframework.kafka.listener.KafkaListenerErrorHandler; + +import org.springframework.kafka.retrytopic.RetryTopicClassLevelIntegrationTests.AbstractFifthTopicListener; +import org.springframework.kafka.retrytopic.RetryTopicClassLevelIntegrationTests.CountDownLatchContainer; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.messaging.Message; +import org.springframework.messaging.converter.CompositeMessageConverter; +import org.springframework.messaging.converter.GenericMessageConverter; +import org.springframework.messaging.converter.SmartMessageConverter; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.retry.annotation.Backoff; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.stereotype.Component; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; +import org.springframework.util.ReflectionUtils; + +import reactor.core.publisher.Mono; + +@SpringJUnitConfig +@DirtiesContext +@EmbeddedKafka(topics = { RetryTopicClassLevelIntegrationTests.FIRST_TOPIC, + RetryTopicClassLevelIntegrationTests.SECOND_TOPIC, + RetryTopicClassLevelIntegrationTests.THIRD_TOPIC, + RetryTopicClassLevelIntegrationTests.FOURTH_TOPIC, + RetryTopicClassLevelIntegrationTests.TWO_LISTENERS_TOPIC, + RetryTopicClassLevelIntegrationTests.MANUAL_TOPIC }) +@TestPropertySource(properties = { "five.attempts=5", "kafka.template=customKafkaTemplate"}) +public class AsyncRetryTopicClassLevelIntegrationTests { + + public final static String FIRST_FUTURE_TOPIC = "myRetryFutureTopic1"; + + public final static String FIRST_FUTURE_LISTENER_ID = "firstFutureTopicId"; + + public final static String FIRST_MONO_TOPIC = "myRetryMonoTopic1"; + + public final static String FIRST_MONO_LISTENER_ID = "firstMonoTopicId"; + + public final static String SECOND_FUTURE_TOPIC = "myRetryFutureTopic2"; + + public final static String SECOND_MONO_TOPIC = "myRetryMonoTopic2"; + + public final static String THIRD_FUTURE_TOPIC = "myRetryFutureTopic3"; + + public final static String THIRD_FUTURE_LISTENER_ID = "thirdFutureTopicId"; + + public final static String THIRD_MONO_TOPIC = "myRetryMonoTopic3"; + + public final static String THIRD_MONO_LISTENER_ID = "thirdMonoTopicId"; + + public final static String FOURTH_FUTURE_TOPIC = "myRetryFutureTopic4"; + + public final static String FOURTH_MONO_TOPIC = "myRetryMonoTopic4"; + + public final static String TWO_LISTENERS_FUTURE_TOPIC = "myRetryFutureTopic5"; + + public final static String TWO_LISTENERS_MONO_TOPIC = "myRetryMonoTopic5"; + + private final static String MAIN_TOPIC_CONTAINER_FACTORY = "kafkaListenerContainerFactory"; + + + public final static String NOT_RETRYABLE_EXCEPTION_FUTURE_TOPIC = "noRetryFutureTopic"; + + public final static String NOT_RETRYABLE_EXCEPTION_MONO_TOPIC = "noRetryMonoTopic"; + + @Autowired + private KafkaTemplate kafkaTemplate; + + @Autowired + private CountDownLatchContainer latchContainer; + + @Autowired + DestinationTopicContainer topicContainer; + + @Test + void shouldRetryFirstFutureTopic(@Autowired KafkaListenerEndpointRegistry registry) { + + kafkaTemplate.send(FIRST_FUTURE_TOPIC, "Testing topic 1"); + assertThat(topicContainer.getNextDestinationTopicFor(FIRST_FUTURE_LISTENER_ID, FIRST_FUTURE_TOPIC).getDestinationName()) + .isEqualTo(FIRST_FUTURE_TOPIC + "-retry"); + assertThat(awaitLatch(latchContainer.futureCountDownLatch1)).isTrue(); + assertThat(awaitLatch(latchContainer.customDltCountdownLatch)).isTrue(); + assertThat(awaitLatch(latchContainer.customMessageConverterCountdownLatch)).isTrue(); + assertThat(awaitLatch(latchContainer.customErrorHandlerCountdownLatch)).isTrue(); + registry.getListenerContainerIds().stream() + .filter(id -> id.startsWith(FIRST_FUTURE_LISTENER_ID)) + .forEach(id -> { + ConcurrentMessageListenerContainer container + = (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); + if (id.equals(FIRST_FUTURE_LISTENER_ID)) { + assertThat(container.getConcurrency()).isEqualTo(2); + } + else { + assertThat(container.getConcurrency()) + .describedAs("Expected %s to have concurrency", id) + .isEqualTo(1); + } + }); + } + + @Test + void shouldRetryFirstMonoTopic(@Autowired KafkaListenerEndpointRegistry registry) { + + kafkaTemplate.send(FIRST_MONO_TOPIC, "Testing topic Mono 1"); + assertThat(topicContainer.getNextDestinationTopicFor(FIRST_MONO_LISTENER_ID, FIRST_MONO_TOPIC).getDestinationName()) + .isEqualTo(FIRST_MONO_TOPIC + "-retry"); + assertThat(awaitLatch(latchContainer.monoCountDownLatch1)).isTrue(); + assertThat(awaitLatch(latchContainer.customDltCountdownLatch)).isTrue(); + assertThat(awaitLatch(latchContainer.customMessageConverterCountdownLatch)).isTrue(); + assertThat(awaitLatch(latchContainer.customErrorHandlerCountdownLatch)).isTrue(); + registry.getListenerContainerIds().stream() + .filter(id -> id.startsWith(FIRST_MONO_LISTENER_ID)) + .forEach(id -> { + ConcurrentMessageListenerContainer container + = (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); + if (id.equals(FIRST_MONO_LISTENER_ID)) { + assertThat(container.getConcurrency()).isEqualTo(2); + } + else { + assertThat(container.getConcurrency()) + .describedAs("Expected %s to have concurrency", id) + .isEqualTo(1); + } + }); + } + + @Test + void shouldRetrySecondFutureTopic() { + kafkaTemplate.send(SECOND_FUTURE_TOPIC, "Testing topic 2"); + assertThat(awaitLatch(latchContainer.futureCountDownLatch2)).isTrue(); + assertThat(awaitLatch(latchContainer.customDltCountdownLatch)).isTrue(); + } + + @Test + void shouldRetrySecondMonoTopic() { + kafkaTemplate.send(SECOND_MONO_TOPIC, "Testing topic 2"); + assertThat(awaitLatch(latchContainer.monoCountDownLatch2)).isTrue(); + assertThat(awaitLatch(latchContainer.customDltCountdownLatch)).isTrue(); + } + + + @Test + void shouldRetryThirdFutureTopicWithTimeout(@Autowired KafkaAdmin admin, + @Autowired KafkaListenerEndpointRegistry registry) throws Exception { + kafkaTemplate.send(THIRD_FUTURE_TOPIC, "Testing topic 3"); + assertThat(awaitLatch(latchContainer.futureCountDownLatch3)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchDltOne)).isTrue(); + Map topics = admin.describeTopics(THIRD_FUTURE_TOPIC, THIRD_FUTURE_TOPIC + "-dlt", FOURTH_FUTURE_TOPIC); + assertThat(topics.get(THIRD_FUTURE_TOPIC).partitions()).hasSize(2); + assertThat(topics.get(THIRD_FUTURE_TOPIC + "-dlt").partitions()).hasSize(3); + assertThat(topics.get(FOURTH_FUTURE_TOPIC).partitions()).hasSize(2); + AtomicReference method = new AtomicReference<>(); + ReflectionUtils.doWithMethods(KafkaAdmin.class, m -> { + m.setAccessible(true); + method.set(m); + }, m -> m.getName().equals("newTopics")); + @SuppressWarnings("unchecked") + Collection weededTopics = (Collection) method.get().invoke(admin); + AtomicInteger weeded = new AtomicInteger(); + weededTopics.forEach(topic -> { + if (topic.name().equals(THIRD_FUTURE_TOPIC) || topic.name().equals(FOURTH_FUTURE_TOPIC)) { + assertThat(topic).isExactlyInstanceOf(NewTopic.class); + weeded.incrementAndGet(); + } + }); + assertThat(weeded.get()).isEqualTo(2); + registry.getListenerContainerIds().stream() + .filter(id -> id.startsWith(THIRD_FUTURE_LISTENER_ID)) + .forEach(id -> { + ConcurrentMessageListenerContainer container = + (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); + if (id.equals(THIRD_FUTURE_LISTENER_ID)) { + assertThat(container.getConcurrency()).isEqualTo(2); + } + else { + assertThat(container.getConcurrency()) + .describedAs("Expected %s to have concurrency", id) + .isEqualTo(1); + } + }); + } + + @Test + void shouldRetryThirdMonoTopicWithTimeout(@Autowired KafkaAdmin admin, + @Autowired KafkaListenerEndpointRegistry registry) throws Exception { + + kafkaTemplate.send(THIRD_MONO_TOPIC, "Testing topic 3"); + assertThat(awaitLatch(latchContainer.monoCountDownLatch3)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchDltOne)).isTrue(); + Map topics = admin.describeTopics(THIRD_MONO_TOPIC, THIRD_MONO_TOPIC + "-dlt", FOURTH_MONO_TOPIC); + assertThat(topics.get(THIRD_MONO_TOPIC).partitions()).hasSize(2); + assertThat(topics.get(THIRD_MONO_TOPIC + "-dlt").partitions()).hasSize(3); + assertThat(topics.get(FOURTH_MONO_TOPIC).partitions()).hasSize(2); + AtomicReference method = new AtomicReference<>(); + ReflectionUtils.doWithMethods(KafkaAdmin.class, m -> { + m.setAccessible(true); + method.set(m); + }, m -> m.getName().equals("newTopics")); + @SuppressWarnings("unchecked") + Collection weededTopics = (Collection) method.get().invoke(admin); + AtomicInteger weeded = new AtomicInteger(); + weededTopics.forEach(topic -> { + if (topic.name().equals(THIRD_MONO_TOPIC) || topic.name().equals(FOURTH_MONO_TOPIC)) { + assertThat(topic).isExactlyInstanceOf(NewTopic.class); + weeded.incrementAndGet(); + } + }); + assertThat(weeded.get()).isEqualTo(2); + registry.getListenerContainerIds().stream() + .filter(id -> id.startsWith(THIRD_MONO_LISTENER_ID)) + .forEach(id -> { + ConcurrentMessageListenerContainer container = + (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); + if (id.equals(THIRD_MONO_LISTENER_ID)) { + assertThat(container.getConcurrency()).isEqualTo(2); + } + else { + assertThat(container.getConcurrency()) + .describedAs("Expected %s to have concurrency", id) + .isEqualTo(1); + } + }); + } + + + @Test + void shouldRetryFourthFutureTopicWithNoDlt() { + kafkaTemplate.send(FOURTH_FUTURE_TOPIC, "Testing topic 4"); + assertThat(awaitLatch(latchContainer.futureCountDownLatch4)).isTrue(); + } + + @Test + void shouldRetryFourthMonoTopicWithNoDlt() { + kafkaTemplate.send(FOURTH_MONO_TOPIC, "Testing topic 4"); + assertThat(awaitLatch(latchContainer.monoCountDownLatch4)).isTrue(); + } + + + + @Test + void shouldRetryFifthTopicWithTwoListenersAndManualAssignment1(@Autowired FifthFutureTopicListener1 listener1, + @Autowired FifthFutureTopicListener2 listener2) { + + kafkaTemplate.send(TWO_LISTENERS_FUTURE_TOPIC, 0, "0", "Testing topic 5 - 0"); + kafkaTemplate.send(TWO_LISTENERS_FUTURE_TOPIC, 1, "0", "Testing topic 5 - 1"); + + assertThat(awaitLatch(latchContainer.countDownLatchDltThree)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatch51)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatch52)).isTrue(); + assertThat(listener1.topics).containsExactly(TWO_LISTENERS_FUTURE_TOPIC, TWO_LISTENERS_FUTURE_TOPIC + + "-listener1-0", TWO_LISTENERS_FUTURE_TOPIC + "-listener1-1", TWO_LISTENERS_FUTURE_TOPIC + "-listener1-2", + TWO_LISTENERS_FUTURE_TOPIC + "-listener1-dlt"); + assertThat(listener2.topics).containsExactly(TWO_LISTENERS_FUTURE_TOPIC, TWO_LISTENERS_FUTURE_TOPIC + + "-listener2-0", TWO_LISTENERS_FUTURE_TOPIC + "-listener2-1", TWO_LISTENERS_FUTURE_TOPIC + "-listener2-2", + TWO_LISTENERS_FUTURE_TOPIC + "-listener2-dlt"); + } + + + + @Test + void shouldGoStraightToDltInFuture() { + kafkaTemplate.send(NOT_RETRYABLE_EXCEPTION_FUTURE_TOPIC, "Testing topic with annotation 1"); + assertThat(awaitLatch(latchContainer.futureCountDownLatchNoRetry)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchDltTwo)).isTrue(); + } + + @Test + void shouldGoStraightToDltInMono() { + kafkaTemplate.send(NOT_RETRYABLE_EXCEPTION_MONO_TOPIC, "Testing topic with annotation 1"); + assertThat(awaitLatch(latchContainer.monoCountDownLatchNoRetry)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchDltTwo)).isTrue(); + } + + private boolean awaitLatch(CountDownLatch latch) { + try { + return latch.await(60, TimeUnit.SECONDS); + } + catch (Exception e) { + fail(e.getMessage()); + throw new RuntimeException(e); + } + } + + @KafkaListener( + id = FIRST_FUTURE_LISTENER_ID, + topics = FIRST_FUTURE_TOPIC, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class FirstFutureTopicListener { + + @Autowired + DestinationTopicContainer topicContainer; + + @Autowired + CountDownLatchContainer container; + + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return CompletableFuture.supplyAsync(() -> { + container.futureCountDownLatch1.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Woooops... in topic " + receivedTopic); + }); + } + } + + + @KafkaListener( + id = FIRST_MONO_LISTENER_ID, + topics = FIRST_MONO_TOPIC, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class FirstTopicMonoListener { + + @Autowired + DestinationTopicContainer topicContainer; + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return Mono.fromCallable(() -> { + container.monoCountDownLatch1.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Woooops... in topic " + receivedTopic); + }).then(); + } + } + + + @KafkaListener(topics = SECOND_FUTURE_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class SecondFutureTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public CompletableFuture listenAgain(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return CompletableFuture.supplyAsync(() -> { + container.countDownIfNotKnown(receivedTopic, container.futureCountDownLatch2); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new IllegalStateException("Another woooops... " + receivedTopic); + }); + } + } + + + @KafkaListener(topics = SECOND_MONO_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class SecondMonoTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listenAgain(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return Mono.fromCallable(() -> { + container.countDownIfNotKnown(receivedTopic, container.monoCountDownLatch2); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new IllegalStateException("Another woooops... " + receivedTopic); + }).then(); + } + } + + + @RetryableTopic( + attempts = "${five.attempts}", + backoff = @Backoff(delay = 250, maxDelay = 1000, multiplier = 1.5), + numPartitions = "#{3}", + timeout = "${missing.property:2000}", + include = MyRetryException.class, kafkaTemplate = "${kafka.template}", + topicSuffixingStrategy = TopicSuffixingStrategy.SUFFIX_WITH_INDEX_VALUE, + concurrency = "1") + @KafkaListener( + id = THIRD_FUTURE_LISTENER_ID, + topics = THIRD_FUTURE_TOPIC, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + concurrency = "2") + static class ThirdFutureTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public CompletableFuture listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return CompletableFuture.supplyAsync(() -> { + container.countDownIfNotKnown(receivedTopic, container.futureCountDownLatch3); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new MyRetryException("Annotated woooops... " + receivedTopic); + }); + } + + @DltHandler + public void annotatedDltMethod(Object message) { + container.countDownLatchDltOne.countDown(); + } + } + + @Component + @RetryableTopic( + attempts = "${five.attempts}", + backoff = @Backoff(delay = 250, maxDelay = 1000, multiplier = 1.5), + numPartitions = "#{3}", + timeout = "${missing.property:2000}", + include = MyRetryException.class, kafkaTemplate = "${kafka.template}", + topicSuffixingStrategy = TopicSuffixingStrategy.SUFFIX_WITH_INDEX_VALUE, + concurrency = "1") + @KafkaListener( + id = THIRD_MONO_LISTENER_ID, + topics = THIRD_MONO_TOPIC, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + concurrency = "2") + static class ThirdMonoTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return Mono.fromCallable(() -> { + container.countDownIfNotKnown(receivedTopic, container.monoCountDownLatch3); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new MyRetryException("Annotated woooops... " + receivedTopic); + }).then(); + } + + @DltHandler + public void annotatedDltMethod(Object message) { + container.countDownLatchDltOne.countDown(); + } + } + + + + @RetryableTopic( + dltStrategy = DltStrategy.NO_DLT, + attempts = "4", + backoff = @Backoff(300), + sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS, + kafkaTemplate = "${kafka.template}") + @KafkaListener(topics = FOURTH_FUTURE_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class FourthFutureTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public CompletableFuture listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return CompletableFuture.supplyAsync(() -> { + container.countDownIfNotKnown(receivedTopic, container.futureCountDownLatch4); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new IllegalStateException("Another woooops... " + receivedTopic); + }); + } + + @DltHandler + public void shouldNotGetHere() { + fail("Dlt should not be processed!"); + } + } + + @RetryableTopic( + dltStrategy = DltStrategy.NO_DLT, + attempts = "4", + backoff = @Backoff(300), + sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS, + kafkaTemplate = "${kafka.template}") + @KafkaListener(topics = FOURTH_MONO_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class FourthMonoTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return Mono.fromCallable(() -> { + container.countDownIfNotKnown(receivedTopic, container.monoCountDownLatch4); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new IllegalStateException("Another woooops... " + receivedTopic); + }).then(); + } + + @DltHandler + public void shouldNotGetHere() { + fail("Dlt should not be processed!"); + } + } + + + static class AbstractFifthTopicListener { + + final List topics = Collections.synchronizedList(new ArrayList<>()); + + @Autowired + CountDownLatchContainer container; + + @DltHandler + public void annotatedDltMethod(ConsumerRecord record) { + this.topics.add(record.topic()); + container.countDownLatchDltThree.countDown(); + } + + } + + @RetryableTopic( + attempts = "4", + backoff = @Backoff(1), + numPartitions = "2", + retryTopicSuffix = "-listener1", + dltTopicSuffix = "-listener1-dlt", + topicSuffixingStrategy = TopicSuffixingStrategy.SUFFIX_WITH_INDEX_VALUE, + sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS, + kafkaTemplate = "${kafka.template}") + @KafkaListener( + id = "fifthTopicId1", + topicPartitions = {@TopicPartition(topic = TWO_LISTENERS_FUTURE_TOPIC, + partitionOffsets = @PartitionOffset(partition = "0", initialOffset = "0"))}, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class FifthFutureTopicListener1 extends AbstractFifthTopicListener { + + @KafkaHandler + public CompletableFuture listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.topics.add(receivedTopic); + container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); + return CompletableFuture.supplyAsync(() -> { + + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Annotated woooops... " + receivedTopic); + }); + } + } + + @RetryableTopic( + attempts = "4", + backoff = @Backoff(1), + numPartitions = "2", + retryTopicSuffix = "-listener2", + dltTopicSuffix = "-listener2-dlt", + topicSuffixingStrategy = TopicSuffixingStrategy.SUFFIX_WITH_INDEX_VALUE, + sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS, + kafkaTemplate = "${kafka.template}") + @KafkaListener( + id = "fifthTopicId2", + topicPartitions = {@TopicPartition(topic = TWO_LISTENERS_FUTURE_TOPIC, + partitionOffsets = @PartitionOffset(partition = "1", initialOffset = "0"))}, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class FifthFutureTopicListener2 extends AbstractFifthTopicListener { + + @KafkaHandler + public CompletableFuture listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.topics.add(receivedTopic); + container.countDownIfNotKnown(receivedTopic, container.countDownLatch52); + return CompletableFuture.supplyAsync(() -> { + + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Annotated woooops... " + receivedTopic); + }); + } + + } + + + @Component + @RetryableTopic(attempts = "3", numPartitions = "3", exclude = MyDontRetryException.class, + backoff = @Backoff(delay = 50, maxDelay = 100, multiplier = 3), + traversingCauses = "true", kafkaTemplate = "${kafka.template}") + @KafkaListener(topics = NOT_RETRYABLE_EXCEPTION_FUTURE_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class NoRetryFutureTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public CompletableFuture listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return CompletableFuture.supplyAsync(() -> { + container.countDownIfNotKnown(receivedTopic, container.futureCountDownLatchNoRetry); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); + }); + } + + @DltHandler + public void annotatedDltMethod(Object message) { + container.countDownLatchDltTwo.countDown(); + } + } + + @Component + @RetryableTopic(attempts = "3", numPartitions = "3", exclude = MyDontRetryException.class, + backoff = @Backoff(delay = 50, maxDelay = 100, multiplier = 3), + traversingCauses = "true", kafkaTemplate = "${kafka.template}") + @KafkaListener(topics = NOT_RETRYABLE_EXCEPTION_MONO_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class NoRetryMonoTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return Mono.fromCallable(() -> { + container.countDownIfNotKnown(receivedTopic, container.monoCountDownLatchNoRetry); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); + }).then(); + } + + @DltHandler + public void annotatedDltMethod(Object message) { + container.countDownLatchDltTwo.countDown(); + } + } + + + @Configuration + static class KafkaProducerConfig { + + @Autowired + EmbeddedKafkaBroker broker; + + @Bean + ProducerFactory producerFactory() { + Map configProps = new HashMap<>(); + configProps.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.broker.getBrokersAsString()); + configProps.put( + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + configProps.put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + return new DefaultKafkaProducerFactory<>(configProps); + } + + @Bean("customKafkaTemplate") + KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); + } + + @Bean + CountDownLatchContainer latchContainer() { + return new CountDownLatchContainer(); + } + } + + @Component + static class CountDownLatchContainer { + + CountDownLatch futureCountDownLatch1 = new CountDownLatch(5); + + CountDownLatch monoCountDownLatch1 = new CountDownLatch(5); + + CountDownLatch futureCountDownLatch2 = new CountDownLatch(3); + + CountDownLatch monoCountDownLatch2 = new CountDownLatch(3); + + CountDownLatch futureCountDownLatch3 = new CountDownLatch(3); + + CountDownLatch monoCountDownLatch3 = new CountDownLatch(3); + + CountDownLatch futureCountDownLatch4 = new CountDownLatch(4); + + CountDownLatch monoCountDownLatch4 = new CountDownLatch(4); + + CountDownLatch countDownLatch51 = new CountDownLatch(4); + + CountDownLatch countDownLatch52 = new CountDownLatch(4); + + CountDownLatch countDownLatch6 = new CountDownLatch(4); + + CountDownLatch futureCountDownLatchNoRetry = new CountDownLatch(1); + + CountDownLatch monoCountDownLatchNoRetry = new CountDownLatch(1); + + CountDownLatch countDownLatchDltOne = new CountDownLatch(1); + + CountDownLatch countDownLatchDltTwo = new CountDownLatch(1); + + CountDownLatch countDownLatchDltThree = new CountDownLatch(2); + + CountDownLatch countDownLatchReuseOne = new CountDownLatch(2); + + CountDownLatch countDownLatchReuseTwo = new CountDownLatch(5); + + CountDownLatch countDownLatchReuseThree = new CountDownLatch(5); + + CountDownLatch customDltCountdownLatch = new CountDownLatch(1); + + CountDownLatch customErrorHandlerCountdownLatch = new CountDownLatch(6); + + CountDownLatch customMessageConverterCountdownLatch = new CountDownLatch(6); + + final List knownTopics = new ArrayList<>(); + + private void countDownIfNotKnown(String receivedTopic, CountDownLatch countDownLatch) { + synchronized (knownTopics) { + if (!knownTopics.contains(receivedTopic)) { + knownTopics.add(receivedTopic); + countDownLatch.countDown(); + } + } + } + } + + @SuppressWarnings("serial") + static class MyRetryException extends RuntimeException { + MyRetryException(String msg) { + super(msg); + } + } + + @SuppressWarnings("serial") + static class MyDontRetryException extends RuntimeException { + MyDontRetryException(String msg) { + super(msg); + } + } + + @Configuration + static class RetryTopicConfigurations extends RetryTopicConfigurationSupport { + + private static final String DLT_METHOD_NAME = "processDltMessage"; + + @Bean + FirstFutureTopicListener firstTopicListener() { + return new FirstFutureTopicListener(); + } + + @Bean + FirstTopicMonoListener firstTopicMonoListener() { + return new FirstTopicMonoListener(); + } + + @Bean + SecondFutureTopicListener secondFutureTopicListener() { + return new SecondFutureTopicListener(); + } + + @Bean + SecondMonoTopicListener secondMonoTopicListener() { + return new SecondMonoTopicListener(); + } + + @Bean + ThirdFutureTopicListener thirdFutureTopicListener() { + return new ThirdFutureTopicListener(); + } + + @Bean + ThirdMonoTopicListener thirdMonoTopicListener() { + return new ThirdMonoTopicListener(); + } + + @Bean + FourthFutureTopicListener fourthFutureTopicListener() { + return new FourthFutureTopicListener(); + } + + @Bean + FourthMonoTopicListener fourthMonoTopicListener() { + return new FourthMonoTopicListener(); + } + + @Bean + NoRetryFutureTopicListener noRetryFutureTopicListener() { + return new NoRetryFutureTopicListener(); + } + + @Bean + NoRetryMonoTopicListener noRetryMonoTopicListener() { + return new NoRetryMonoTopicListener(); + } + + @Bean + FifthFutureTopicListener1 fifthFutureTopicListener1() { + return new FifthFutureTopicListener1(); + } + + @Bean + FifthFutureTopicListener2 fifthFutureTopicListener2() { + return new FifthFutureTopicListener2(); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor() { + return new MyCustomDltProcessor(); + } + + @Bean + TaskScheduler taskScheduler() { + return new ThreadPoolTaskScheduler(); + } + + @Bean + KafkaListenerErrorHandler myCustomErrorHandler( + CountDownLatchContainer container) { + return (message, exception) -> { + container.customErrorHandlerCountdownLatch.countDown(); + throw exception; + }; + } + + @Bean + SmartMessageConverter myCustomMessageConverter( + CountDownLatchContainer container) { + return new CompositeMessageConverter(Collections.singletonList(new GenericMessageConverter())) { + + @Override + public Object fromMessage(Message message, Class targetClass, Object conversionHint) { + container.customMessageConverterCountdownLatch.countDown(); + return super.fromMessage(message, targetClass, conversionHint); + } + }; + } + + + @Bean + RetryTopicConfiguration firstRetryTopic(KafkaTemplate template) { + return RetryTopicConfigurationBuilder + .newInstance() + .fixedBackOff(50) + .maxAttempts(5) + .concurrency(1) + .useSingleTopicForSameIntervals() + .includeTopic(FIRST_FUTURE_TOPIC) + .doNotRetryOnDltFailure() + .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) + .create(template); + } + + @Bean + RetryTopicConfiguration firstRetryMonoTopic(KafkaTemplate template) { + return RetryTopicConfigurationBuilder + .newInstance() + .fixedBackOff(50) + .maxAttempts(5) + .concurrency(1) + .useSingleTopicForSameIntervals() + .includeTopic(FIRST_MONO_TOPIC) + .doNotRetryOnDltFailure() + .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) + .create(template); + } + + @Bean + RetryTopicConfiguration secondRetryFutureTopic(KafkaTemplate template) { + return RetryTopicConfigurationBuilder + .newInstance() + .exponentialBackoff(500, 2, 10000) + .retryOn(Arrays.asList(IllegalStateException.class, IllegalAccessException.class)) + .traversingCauses() + .includeTopic(SECOND_FUTURE_TOPIC) + .doNotRetryOnDltFailure() + .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) + .create(template); + } + + @Bean + RetryTopicConfiguration secondRetryMonoTopic(KafkaTemplate template) { + return RetryTopicConfigurationBuilder + .newInstance() + .exponentialBackoff(500, 2, 10000) + .retryOn(Arrays.asList(IllegalStateException.class, IllegalAccessException.class)) + .traversingCauses() + .includeTopic(SECOND_MONO_TOPIC) + .doNotRetryOnDltFailure() + .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) + .create(template); + } + + + @Bean + NewTopics topics() { + + NewTopic thirdFutureTopic = TopicBuilder.name(THIRD_FUTURE_TOPIC).partitions(2).replicas(1).build(); + NewTopic thirdMonoTopic = TopicBuilder.name(THIRD_MONO_TOPIC).partitions(2).replicas(1).build(); + NewTopic fourthFutureTopic = TopicBuilder.name(FOURTH_FUTURE_TOPIC).partitions(2).replicas(1).build(); + NewTopic fourthMonoTopic = TopicBuilder.name(FOURTH_MONO_TOPIC).partitions(2).replicas(1).build(); + + return new NewTopics( + thirdFutureTopic, + thirdMonoTopic, + fourthFutureTopic, + fourthMonoTopic); + } + } + + @EnableKafka + @Configuration + static class KafkaConsumerConfig { + + @Autowired + EmbeddedKafkaBroker broker; + + @Bean + KafkaAdmin kafkaAdmin() { + Map configs = new HashMap<>(); + configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, this.broker.getBrokersAsString()); + return new KafkaAdmin(configs); + } + +// @Bean +// NewTopic topic() { +// return TopicBuilder.name(THIRD_TOPIC).partitions(2).replicas(1).build(); +// } +// +// @Bean +// NewTopics topics() { +// return new NewTopics(TopicBuilder.name(FOURTH_TOPIC).partitions(2).replicas(1).build()); +// } + + @Bean + ConsumerFactory consumerFactory() { + Map props = new HashMap<>(); + props.put( + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.broker.getBrokersAsString()); + props.put( + ConsumerConfig.GROUP_ID_CONFIG, + "groupId"); + props.put( + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class); + props.put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class); + props.put( + ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + + return new DefaultKafkaConsumerFactory<>(props); + } + + @Bean + ConcurrentKafkaListenerContainerFactory retryTopicListenerContainerFactory( + ConsumerFactory consumerFactory) { + + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + ContainerProperties props = factory.getContainerProperties(); + props.setIdleEventInterval(100L); + props.setPollTimeout(50L); + props.setIdlePartitionEventInterval(100L); + factory.setConsumerFactory(consumerFactory); + factory.setConcurrency(1); + factory.setContainerCustomizer( + container -> container.getContainerProperties().setIdlePartitionEventInterval(100L)); + return factory; + } + + @Bean + ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory( + ConsumerFactory consumerFactory) { + + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(consumerFactory); + factory.setConcurrency(1); + factory.setContainerCustomizer(container -> { + if (container.getListenerId().startsWith("manual")) { + container.getContainerProperties().setAckMode(AckMode.MANUAL); + container.getContainerProperties().setAsyncAcks(true); + } + }); + return factory; + } + +// @Bean +// TaskScheduler sched() { +// return new ThreadPoolTaskScheduler(); +// } + + } + + @Component + static class MyCustomDltProcessor { + + @Autowired + KafkaTemplate kafkaTemplate; + + @Autowired + CountDownLatchContainer container; + + public void processDltMessage(Object message) { + container.customDltCountdownLatch.countDown(); + throw new RuntimeException("Dlt Error!"); + } + } +} From 87ac522e8b3edc18b1a5c8c332d4ff5a85ca526a Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 1 Oct 2024 21:42:00 +0900 Subject: [PATCH 03/30] GH-3276: Draft and test codes. --- .../kafka/core/FailedRecordTuple.java | 32 + .../KafkaMessageListenerContainer.java | 36 +- .../kafka/listener/MessageListener.java | 6 +- ...fkaBackoffAwareMessageListenerAdapter.java | 17 +- .../MessagingMessageListenerAdapter.java | 13 +- ...RetryTopicClassLevelIntegrationTests.java} | 886 +++++++--------- ...eRetryTopicClassLevelIntegrationTests.java | 958 ++++++++++++++++++ 7 files changed, 1413 insertions(+), 535 deletions(-) create mode 100644 spring-kafka/src/main/java/org/springframework/kafka/core/FailedRecordTuple.java rename spring-kafka/src/test/java/org/springframework/kafka/retrytopic/{AsyncRetryTopicClassLevelIntegrationTests.java => AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java} (52%) create mode 100644 spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java diff --git a/spring-kafka/src/main/java/org/springframework/kafka/core/FailedRecordTuple.java b/spring-kafka/src/main/java/org/springframework/kafka/core/FailedRecordTuple.java new file mode 100644 index 0000000000..acf9547708 --- /dev/null +++ b/spring-kafka/src/main/java/org/springframework/kafka/core/FailedRecordTuple.java @@ -0,0 +1,32 @@ +/* + * Copyright 2016-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * ... + * @author Sanghyeok An + * + * @since 3.3 + */ + +package org.springframework.kafka.core; + +import org.apache.kafka.clients.consumer.ConsumerRecord; + +public record FailedRecordTuple( + ConsumerRecord record, + RuntimeException ex) { + +}; diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index 7bb04c4b21..b5c4fd5c2e 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -16,7 +16,6 @@ package org.springframework.kafka.listener; -import java.lang.Thread.UncaughtExceptionHandler; import java.nio.ByteBuffer; import java.time.Duration; import java.util.AbstractMap.SimpleEntry; @@ -39,6 +38,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledFuture; @@ -84,6 +84,7 @@ import org.springframework.core.task.SimpleAsyncTaskExecutor; import org.springframework.kafka.KafkaException; import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.FailedRecordTuple; import org.springframework.kafka.core.KafkaAdmin; import org.springframework.kafka.core.KafkaResourceHolder; import org.springframework.kafka.event.ConsumerFailedToStartEvent; @@ -107,6 +108,7 @@ import org.springframework.kafka.listener.ContainerProperties.AckMode; import org.springframework.kafka.listener.ContainerProperties.AssignmentCommitOption; import org.springframework.kafka.listener.ContainerProperties.EOSMode; +import org.springframework.kafka.listener.FailedRecordTracker.FailedRecord; import org.springframework.kafka.listener.adapter.AsyncRepliesAware; import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.KafkaHeaders; @@ -842,6 +844,9 @@ private final class ListenerConsumer implements SchedulingAwareRunnable, Consume private volatile long lastPoll = System.currentTimeMillis(); + private final ConcurrentLinkedDeque> failedRecords = new ConcurrentLinkedDeque(); + + @SuppressWarnings(UNCHECKED) ListenerConsumer(GenericMessageListener listener, ListenerType listenerType, ObservationRegistry observationRegistry) { @@ -899,10 +904,9 @@ else if (listener instanceof MessageListener) { this.observationEnabled = this.containerProperties.isObservationEnabled(); if (!AopUtils.isAopProxy(listener)) { - final BiConsumer, RuntimeException> asyncRetryCallback - = (cRecord, runtimeException) -> - this.invokeErrorHandlerBySingleRecord(cRecord, runtimeException); - this.listener.setAsyncRetryCallback(asyncRetryCallback); + final java.util.function.Consumer callbackForAsyncFailureQueue = + (fRecord) -> this.failedRecords.addLast(fRecord); + this.listener.setCallbackForAsyncFailureQueue(callbackForAsyncFailureQueue); } } else { @@ -1303,6 +1307,15 @@ public void run() { boolean failedAuthRetry = false; this.lastReceive = System.currentTimeMillis(); while (isRunning()) { + + try { + handleAsyncFailure(); + } catch (Exception e) { + // TODO: Need to improve error handling. + // TODO: Need to determine how to handle a failed message. + logger.error("Failed to process re-try messages. "); + } + try { pollAndInvoke(); if (failedAuthRetry) { @@ -1444,6 +1457,19 @@ protected void pollAndInvoke() { } } + protected void handleAsyncFailure() { + List copyFailedRecords = new ArrayList<>(); + while (!this.failedRecords.isEmpty()) { + FailedRecordTuple failedRecordTuple = this.failedRecords.pollFirst(); + copyFailedRecords.add(failedRecordTuple); + } + + if (!copyFailedRecords.isEmpty()) { + copyFailedRecords.forEach(failedRecordTuple -> + this.invokeErrorHandlerBySingleRecord(failedRecordTuple.record(), failedRecordTuple.ex())); + } + } + private void doProcessCommits() { if (!this.autoCommit && !this.isRecordAck) { try { diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java index ccce3541e7..388478a492 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java @@ -16,9 +16,10 @@ package org.springframework.kafka.listener; -import java.util.function.BiConsumer; +import java.util.function.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.springframework.kafka.core.FailedRecordTuple; /** * Listener for handling individual incoming Kafka messages. @@ -28,11 +29,12 @@ * * @author Marius Bogoevici * @author Gary Russell + * @author Sanghyeok An */ @FunctionalInterface public interface MessageListener extends GenericMessageListener> { - default void setAsyncRetryCallback(BiConsumer, RuntimeException> asyncRetryCallback) { + default void setCallbackForAsyncFailureQueue(Consumer asyncRetryCallback) { // } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java index 74ea973dab..ce38a17cfd 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java @@ -26,6 +26,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.TopicPartition; +import org.springframework.kafka.core.FailedRecordTuple; import org.springframework.kafka.listener.AcknowledgingConsumerAwareMessageListener; import org.springframework.kafka.listener.KafkaBackoffException; import org.springframework.kafka.listener.KafkaConsumerBackoffManager; @@ -44,6 +45,7 @@ * @param the record key type. * @param the record value type. * @author Tomaz Fernandes + * @author Sanghyeok An * @since 2.7 * */ @@ -94,17 +96,6 @@ public void onMessage(ConsumerRecord consumerRecord, @Nullable Acknowledgm maybeGetBackoffTimestamp(consumerRecord) .ifPresent(nextExecutionTimestamp -> this.kafkaConsumerBackoffManager .backOffIfNecessary(createContext(consumerRecord, nextExecutionTimestamp, consumer))); - // ConsumerRecord(topic = myRetryFutureTopic5, partition = 1, leaderEpoch = 0, offset = 0, CreateTime = 1727697786519, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [], isReadOnly = false), key = 0, value = Testing topic 5 - 1) - // ConsumerRecord(topic = myRetryFutureTopic5-listener2-0, partition = 1, leaderEpoch = 0, offset = 0, CreateTime = 1727697787141, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 83, 116, 97, 103, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 56, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 119, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 50, 51, 50, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 56, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 105, 110, 118, 111, 107, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 51, 57, 50, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 56, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 53, 48, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 105, 110, 118, 111, 107, 101, 68, 101, 108, 101, 103, 97, 116, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 49, 49, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 49, 48, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 53, 48, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 56, 50, 49, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 57, 57, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 100, 111, 73, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 36, 53, 52, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 49, 55, 41, 10, 9, 97, 116, 32, 105, 111, 46, 109, 105, 99, 114, 111, 109, 101, 116, 101, 114, 46, 111, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 79, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 111, 98, 115, 101, 114, 118, 101, 40, 79, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 106, 97, 118, 97, 58, 53, 54, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 49, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 87, 105, 116, 104, 82, 101, 99, 111, 114, 100, 115, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 53, 53, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 52, 52, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 48, 57, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 73, 102, 72, 97, 118, 101, 82, 101, 99, 111, 114, 100, 115, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 52, 55, 51, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 112, 111, 108, 108, 65, 110, 100, 73, 110, 118, 111, 107, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 52, 51, 56, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 114, 117, 110, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 51, 48, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 82, 117, 110, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 56, 48, 52, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 56, 51, 51, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 53, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10]), RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 1]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -105]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21])], isReadOnly = false), key = 0, value = Testing topic 5 - 1) - // ConsumerRecord(topic = myRetryFutureTopic5-listener2-1, partition = 1, leaderEpoch = 0, offset = 0, CreateTime = 1727697788166, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 1]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -105]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 48]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 48, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 53, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -19])], isReadOnly = false), key = 0, value = Testing topic 5 - 1) - // ConsumerRecord(topic = myRetryFutureTopic5-listener2-2, partition = 1, leaderEpoch = 0, offset = 0, CreateTime = 1727697789173, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 1]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -105]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -19]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 49]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 49, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 53, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 4]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -35])], isReadOnly = false), key = 0, value = Testing topic 5 - 1) - // ConsumerRecord(topic = myRetryFutureTopic5-listener2-dlt, partition = 1, leaderEpoch = 0, offset = 0, CreateTime = 1727697790187, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 1]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -105]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -19]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 4]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -35]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 50]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 64, 55, 49, 56, 57, 49, 100, 54, 98, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 50, 45, 50, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 50, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 50, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 53, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -105]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 5]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -22])], isReadOnly = false), key = 0, value = Testing topic 5 - 1) - - // ConsumerRecord(topic = myRetryFutureTopic5, partition = 0, leaderEpoch = 0, offset = 0, CreateTime = 1727697786516, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [], isReadOnly = false), key = 0, value = Testing topic 5 - 0) - // ConsumerRecord(topic = myRetryFutureTopic5-listener1-0, partition = 0, leaderEpoch = 0, offset = 0, CreateTime = 1727697787141, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 83, 116, 97, 103, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 56, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 119, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 50, 51, 50, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 56, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 105, 110, 118, 111, 107, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 51, 57, 50, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 56, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 82, 101, 99, 111, 114, 100, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 53, 48, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 105, 110, 118, 111, 107, 101, 68, 101, 108, 101, 103, 97, 116, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 49, 49, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 49, 48, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 111, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 66, 97, 99, 107, 111, 102, 102, 65, 119, 97, 114, 101, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 53, 48, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 56, 50, 49, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 79, 110, 77, 101, 115, 115, 97, 103, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 57, 57, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 100, 111, 73, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 36, 53, 52, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 49, 55, 41, 10, 9, 97, 116, 32, 105, 111, 46, 109, 105, 99, 114, 111, 109, 101, 116, 101, 114, 46, 111, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 79, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 111, 98, 115, 101, 114, 118, 101, 40, 79, 98, 115, 101, 114, 118, 97, 116, 105, 111, 110, 46, 106, 97, 118, 97, 58, 53, 54, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 55, 49, 53, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 100, 111, 73, 110, 118, 111, 107, 101, 87, 105, 116, 104, 82, 101, 99, 111, 114, 100, 115, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 53, 53, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 82, 101, 99, 111, 114, 100, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 52, 52, 54, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 76, 105, 115, 116, 101, 110, 101, 114, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 50, 48, 57, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 105, 110, 118, 111, 107, 101, 73, 102, 72, 97, 118, 101, 82, 101, 99, 111, 114, 100, 115, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 52, 55, 51, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 112, 111, 108, 108, 65, 110, 100, 73, 110, 118, 111, 107, 101, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 52, 51, 56, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 36, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 115, 117, 109, 101, 114, 46, 114, 117, 110, 40, 75, 97, 102, 107, 97, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 67, 111, 110, 116, 97, 105, 110, 101, 114, 46, 106, 97, 118, 97, 58, 49, 51, 48, 55, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 82, 117, 110, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 56, 48, 52, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 56, 51, 51, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 50, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10]), RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 0]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -108]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21])], isReadOnly = false), key = 0, value = Testing topic 5 - 0) - // ConsumerRecord(topic = myRetryFutureTopic5-listener1-1, partition = 0, leaderEpoch = 0, offset = 0, CreateTime = 1727697788165, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 0]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -108]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 48]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 48, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 50, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -20])], isReadOnly = false), key = 0, value = Testing topic 5 - 0) - // ConsumerRecord(topic = myRetryFutureTopic5-listener1-2, partition = 0, leaderEpoch = 0, offset = 0, CreateTime = 1727697789173, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 0]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -108]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -20]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 49]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 49, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 50, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 4]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -35])], isReadOnly = false), key = 0, value = Testing topic 5 - 0) - // ConsumerRecord(topic = myRetryFutureTopic5-listener1-dlt, partition = 0, leaderEpoch = 0, offset = 0, CreateTime = 1727697790187, serialized key size = 1, serialized value size = 19, headers = RecordHeaders(headers = [RecordHeader(key = kafka_original-topic, value = [109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53]), RecordHeader(key = kafka_original-partition, value = [0, 0, 0, 0]), RecordHeader(key = kafka_original-offset, value = [0, 0, 0, 0, 0, 0, 0, 0]), RecordHeader(key = kafka_original-timestamp, value = [0, 0, 1, -110, 66, -48, 22, -108]), RecordHeader(key = kafka_original-timestamp-type, value = [67, 114, 101, 97, 116, 101, 84, 105, 109, 101]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 2]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 28, -21]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 3]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 32, -20]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 4]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -35]), RecordHeader(key = kafka_exception-fqcn, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-cause-fqcn, value = [106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110]), RecordHeader(key = kafka_exception-message, value = [65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 59, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 50]), RecordHeader(key = kafka_exception-stacktrace, value = [111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 76, 105, 115, 116, 101, 110, 101, 114, 69, 120, 101, 99, 117, 116, 105, 111, 110, 70, 97, 105, 108, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 115, 121, 110, 99, 32, 70, 97, 105, 108, 10, 69, 110, 100, 112, 111, 105, 110, 116, 32, 104, 97, 110, 100, 108, 101, 114, 32, 100, 101, 116, 97, 105, 108, 115, 58, 10, 77, 101, 116, 104, 111, 100, 32, 91, 112, 117, 98, 108, 105, 99, 32, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 60, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 86, 111, 105, 100, 62, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 40, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 41, 93, 10, 66, 101, 97, 110, 32, 91, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 64, 52, 52, 101, 48, 99, 51, 100, 93, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 97, 115, 121, 110, 99, 70, 97, 105, 108, 117, 114, 101, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 54, 55, 55, 41, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 108, 105, 115, 116, 101, 110, 101, 114, 46, 97, 100, 97, 112, 116, 101, 114, 46, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 108, 97, 109, 98, 100, 97, 36, 104, 97, 110, 100, 108, 101, 82, 101, 115, 117, 108, 116, 36, 49, 40, 77, 101, 115, 115, 97, 103, 105, 110, 103, 77, 101, 115, 115, 97, 103, 101, 76, 105, 115, 116, 101, 110, 101, 114, 65, 100, 97, 112, 116, 101, 114, 46, 106, 97, 118, 97, 58, 52, 57, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 117, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 54, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 85, 110, 105, 87, 104, 101, 110, 67, 111, 109, 112, 108, 101, 116, 101, 46, 116, 114, 121, 70, 105, 114, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 56, 52, 49, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 112, 111, 115, 116, 67, 111, 109, 112, 108, 101, 116, 101, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 53, 49, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 101, 120, 101, 99, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 48, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 100, 111, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 84, 97, 115, 107, 46, 106, 97, 118, 97, 58, 51, 55, 51, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 36, 87, 111, 114, 107, 81, 117, 101, 117, 101, 46, 116, 111, 112, 76, 101, 118, 101, 108, 69, 120, 101, 99, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 49, 56, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 115, 99, 97, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 53, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 114, 117, 110, 87, 111, 114, 107, 101, 114, 40, 70, 111, 114, 107, 74, 111, 105, 110, 80, 111, 111, 108, 46, 106, 97, 118, 97, 58, 49, 54, 50, 50, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 114, 117, 110, 40, 70, 111, 114, 107, 74, 111, 105, 110, 87, 111, 114, 107, 101, 114, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 58, 49, 54, 53, 41, 10, 67, 97, 117, 115, 101, 100, 32, 98, 121, 58, 32, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 58, 32, 65, 110, 110, 111, 116, 97, 116, 101, 100, 32, 119, 111, 111, 111, 111, 112, 115, 46, 46, 46, 32, 109, 121, 82, 101, 116, 114, 121, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 53, 45, 108, 105, 115, 116, 101, 110, 101, 114, 49, 45, 50, 10, 9, 97, 116, 32, 111, 114, 103, 46, 115, 112, 114, 105, 110, 103, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 107, 97, 102, 107, 97, 46, 114, 101, 116, 114, 121, 116, 111, 112, 105, 99, 46, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 36, 70, 105, 102, 116, 104, 70, 117, 116, 117, 114, 101, 84, 111, 112, 105, 99, 76, 105, 115, 116, 101, 110, 101, 114, 49, 46, 108, 97, 109, 98, 100, 97, 36, 108, 105, 115, 116, 101, 110, 87, 105, 116, 104, 65, 110, 110, 111, 116, 97, 116, 105, 111, 110, 36, 48, 40, 65, 115, 121, 110, 99, 82, 101, 116, 114, 121, 84, 111, 112, 105, 99, 67, 108, 97, 115, 115, 76, 101, 118, 101, 108, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 84, 101, 115, 116, 115, 46, 106, 97, 118, 97, 58, 54, 50, 53, 41, 10, 9, 97, 116, 32, 106, 97, 118, 97, 46, 98, 97, 115, 101, 47, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 36, 65, 115, 121, 110, 99, 83, 117, 112, 112, 108, 121, 46, 114, 117, 110, 40, 67, 111, 109, 112, 108, 101, 116, 97, 98, 108, 101, 70, 117, 116, 117, 114, 101, 46, 106, 97, 118, 97, 58, 49, 55, 54, 56, 41, 10, 9, 46, 46, 46, 32, 54, 32, 109, 111, 114, 101, 10]), RecordHeader(key = retry_topic-original-timestamp, value = [1, -110, 66, -48, 22, -108]), RecordHeader(key = retry_topic-attempts, value = [0, 0, 0, 5]), RecordHeader(key = retry_topic-backoff-timestamp, value = [1, -110, 66, -48, 36, -22])], isReadOnly = false), key = 0, value = Testing topic 5 - 0) try { invokeDelegateOnMessage(consumerRecord, acknowledgment, consumer); } @@ -157,7 +148,7 @@ public void onMessage(ConsumerRecord data, Consumer consumer) { } @Override - public void setAsyncRetryCallback(BiConsumer, RuntimeException> asyncRetryCallback) { - this.delegate.setAsyncRetryCallback(asyncRetryCallback); + public void setCallbackForAsyncFailureQueue(java.util.function.Consumer callbackForAsyncFailureQueue) { + this.delegate.setCallbackForAsyncFailureQueue(callbackForAsyncFailureQueue); } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index 4091a3c271..fed47f5f56 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -47,8 +47,8 @@ import org.springframework.expression.spel.standard.SpelExpressionParser; import org.springframework.expression.spel.support.StandardEvaluationContext; import org.springframework.expression.spel.support.StandardTypeConverter; +import org.springframework.kafka.core.FailedRecordTuple; import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.listener.AsyncRetryableException; import org.springframework.kafka.listener.ConsumerSeekAware; import org.springframework.kafka.listener.KafkaListenerErrorHandler; import org.springframework.kafka.listener.ListenerExecutionFailedException; @@ -94,6 +94,7 @@ * @author Wang ZhiYang * @author Huijin Hong * @author Soby Chacko + * @author Sanghyeok An */ public abstract class MessagingMessageListenerAdapter implements ConsumerSeekAware, AsyncRepliesAware { @@ -157,6 +158,8 @@ public abstract class MessagingMessageListenerAdapter implements ConsumerS private BiConsumer, RuntimeException> asyncRetryCallback; + private java.util.function.Consumer callbackForAsyncFailureQueue; + /** * Create an instance with the provided bean and method. * @param bean the bean. @@ -164,6 +167,7 @@ public abstract class MessagingMessageListenerAdapter implements ConsumerS */ protected MessagingMessageListenerAdapter(Object bean, Method method) { this(bean, method, null); + System.out.println("here"); } /** @@ -689,7 +693,8 @@ protected void asyncFailure(Object request, @Nullable Acknowledgment acknowledgm if (request instanceof ConsumerRecord && ex instanceof RuntimeException) { ConsumerRecord record = (ConsumerRecord) request; - asyncRetryCallback.accept(record, (RuntimeException) ex); + FailedRecordTuple failedRecordTuple = new FailedRecordTuple(record, (RuntimeException) ex); + this.callbackForAsyncFailureQueue.accept(failedRecordTuple); } } } @@ -910,4 +915,8 @@ public void setAsyncRetryCallback(BiConsumer, RuntimeExcept this.asyncRetryCallback = asyncRetryCallback; } + public void putInAsyncFailureQueue(java.util.function.Consumer callbackForAsyncFailureQueue) { + this.callbackForAsyncFailureQueue = callbackForAsyncFailureQueue; + } + } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java similarity index 52% rename from spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncRetryTopicClassLevelIntegrationTests.java rename to spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java index 97a629b3ec..45f4d3947c 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java @@ -1,7 +1,24 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.springframework.kafka.retrytopic; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; +import static org.awaitility.Awaitility.await; import java.lang.reflect.Method; import java.util.ArrayList; @@ -11,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -20,11 +38,14 @@ import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; +import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; @@ -35,7 +56,6 @@ import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.annotation.PartitionOffset; import org.springframework.kafka.annotation.RetryableTopic; -import org.springframework.kafka.annotation.TopicPartition; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; import org.springframework.kafka.config.KafkaListenerEndpointRegistry; import org.springframework.kafka.config.TopicBuilder; @@ -46,14 +66,11 @@ import org.springframework.kafka.core.KafkaAdmin.NewTopics; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.core.ProducerFactory; -import org.springframework.kafka.listener.AsyncRetryableException; import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; import org.springframework.kafka.listener.ContainerProperties; import org.springframework.kafka.listener.ContainerProperties.AckMode; import org.springframework.kafka.listener.KafkaListenerErrorHandler; - -import org.springframework.kafka.retrytopic.RetryTopicClassLevelIntegrationTests.AbstractFifthTopicListener; -import org.springframework.kafka.retrytopic.RetryTopicClassLevelIntegrationTests.CountDownLatchContainer; +import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.KafkaHeaders; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.context.EmbeddedKafka; @@ -65,60 +82,49 @@ import org.springframework.retry.annotation.Backoff; import org.springframework.scheduling.TaskScheduler; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; -import org.springframework.stereotype.Component; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; -import org.springframework.util.ReflectionUtils; -import reactor.core.publisher.Mono; + +/** + * @author Sanghyeok An + * @since 3.3.0 + */ @SpringJUnitConfig @DirtiesContext -@EmbeddedKafka(topics = { RetryTopicClassLevelIntegrationTests.FIRST_TOPIC, - RetryTopicClassLevelIntegrationTests.SECOND_TOPIC, - RetryTopicClassLevelIntegrationTests.THIRD_TOPIC, - RetryTopicClassLevelIntegrationTests.FOURTH_TOPIC, - RetryTopicClassLevelIntegrationTests.TWO_LISTENERS_TOPIC, - RetryTopicClassLevelIntegrationTests.MANUAL_TOPIC }) +@EmbeddedKafka(topics = { AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.FIRST_TOPIC, + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.SECOND_TOPIC, + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.THIRD_TOPIC, + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.FOURTH_TOPIC, + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.TWO_LISTENERS_TOPIC, + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.MANUAL_TOPIC }) @TestPropertySource(properties = { "five.attempts=5", "kafka.template=customKafkaTemplate"}) -public class AsyncRetryTopicClassLevelIntegrationTests { - - public final static String FIRST_FUTURE_TOPIC = "myRetryFutureTopic1"; - - public final static String FIRST_FUTURE_LISTENER_ID = "firstFutureTopicId"; - - public final static String FIRST_MONO_TOPIC = "myRetryMonoTopic1"; - - public final static String FIRST_MONO_LISTENER_ID = "firstMonoTopicId"; +public class AsyncCompletableFutureRetryTopicClassLevelIntegrationTests { - public final static String SECOND_FUTURE_TOPIC = "myRetryFutureTopic2"; + public final static String FIRST_TOPIC = "myRetryTopic1"; - public final static String SECOND_MONO_TOPIC = "myRetryMonoTopic2"; + public final static String SECOND_TOPIC = "myRetryTopic2"; - public final static String THIRD_FUTURE_TOPIC = "myRetryFutureTopic3"; + public final static String THIRD_TOPIC = "myRetryTopic3"; - public final static String THIRD_FUTURE_LISTENER_ID = "thirdFutureTopicId"; + public final static String FOURTH_TOPIC = "myRetryTopic4"; - public final static String THIRD_MONO_TOPIC = "myRetryMonoTopic3"; + public final static String TWO_LISTENERS_TOPIC = "myRetryTopic5"; - public final static String THIRD_MONO_LISTENER_ID = "thirdMonoTopicId"; + public final static String MANUAL_TOPIC = "myRetryTopic6"; - public final static String FOURTH_FUTURE_TOPIC = "myRetryFutureTopic4"; + public final static String NOT_RETRYABLE_EXCEPTION_TOPIC = "noRetryTopic"; - public final static String FOURTH_MONO_TOPIC = "myRetryMonoTopic4"; + public final static String FIRST_REUSE_RETRY_TOPIC = "reuseRetry1"; - public final static String TWO_LISTENERS_FUTURE_TOPIC = "myRetryFutureTopic5"; + public final static String SECOND_REUSE_RETRY_TOPIC = "reuseRetry2"; - public final static String TWO_LISTENERS_MONO_TOPIC = "myRetryMonoTopic5"; + public final static String THIRD_REUSE_RETRY_TOPIC = "reuseRetry3"; private final static String MAIN_TOPIC_CONTAINER_FACTORY = "kafkaListenerContainerFactory"; - - public final static String NOT_RETRYABLE_EXCEPTION_FUTURE_TOPIC = "noRetryFutureTopic"; - - public final static String NOT_RETRYABLE_EXCEPTION_MONO_TOPIC = "noRetryMonoTopic"; - @Autowired private KafkaTemplate kafkaTemplate; @@ -129,47 +135,20 @@ public class AsyncRetryTopicClassLevelIntegrationTests { DestinationTopicContainer topicContainer; @Test - void shouldRetryFirstFutureTopic(@Autowired KafkaListenerEndpointRegistry registry) { - - kafkaTemplate.send(FIRST_FUTURE_TOPIC, "Testing topic 1"); - assertThat(topicContainer.getNextDestinationTopicFor(FIRST_FUTURE_LISTENER_ID, FIRST_FUTURE_TOPIC).getDestinationName()) - .isEqualTo(FIRST_FUTURE_TOPIC + "-retry"); - assertThat(awaitLatch(latchContainer.futureCountDownLatch1)).isTrue(); + void shouldRetryFirstTopic(@Autowired KafkaListenerEndpointRegistry registry) { + kafkaTemplate.send(FIRST_TOPIC, "Testing topic 1"); + assertThat(topicContainer.getNextDestinationTopicFor("firstTopicId", FIRST_TOPIC).getDestinationName()) + .isEqualTo("myRetryTopic1-retry"); + assertThat(awaitLatch(latchContainer.countDownLatch1)).isTrue(); assertThat(awaitLatch(latchContainer.customDltCountdownLatch)).isTrue(); - assertThat(awaitLatch(latchContainer.customMessageConverterCountdownLatch)).isTrue(); assertThat(awaitLatch(latchContainer.customErrorHandlerCountdownLatch)).isTrue(); - registry.getListenerContainerIds().stream() - .filter(id -> id.startsWith(FIRST_FUTURE_LISTENER_ID)) - .forEach(id -> { - ConcurrentMessageListenerContainer container - = (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); - if (id.equals(FIRST_FUTURE_LISTENER_ID)) { - assertThat(container.getConcurrency()).isEqualTo(2); - } - else { - assertThat(container.getConcurrency()) - .describedAs("Expected %s to have concurrency", id) - .isEqualTo(1); - } - }); - } - - @Test - void shouldRetryFirstMonoTopic(@Autowired KafkaListenerEndpointRegistry registry) { - - kafkaTemplate.send(FIRST_MONO_TOPIC, "Testing topic Mono 1"); - assertThat(topicContainer.getNextDestinationTopicFor(FIRST_MONO_LISTENER_ID, FIRST_MONO_TOPIC).getDestinationName()) - .isEqualTo(FIRST_MONO_TOPIC + "-retry"); - assertThat(awaitLatch(latchContainer.monoCountDownLatch1)).isTrue(); - assertThat(awaitLatch(latchContainer.customDltCountdownLatch)).isTrue(); assertThat(awaitLatch(latchContainer.customMessageConverterCountdownLatch)).isTrue(); - assertThat(awaitLatch(latchContainer.customErrorHandlerCountdownLatch)).isTrue(); registry.getListenerContainerIds().stream() - .filter(id -> id.startsWith(FIRST_MONO_LISTENER_ID)) + .filter(id -> id.startsWith("first")) .forEach(id -> { - ConcurrentMessageListenerContainer container - = (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); - if (id.equals(FIRST_MONO_LISTENER_ID)) { + ConcurrentMessageListenerContainer container = (ConcurrentMessageListenerContainer) registry + .getListenerContainer(id); + if (id.equals("firstTopicId")) { assertThat(container.getConcurrency()).isEqualTo(2); } else { @@ -181,74 +160,25 @@ void shouldRetryFirstMonoTopic(@Autowired KafkaListenerEndpointRegistry registry } @Test - void shouldRetrySecondFutureTopic() { - kafkaTemplate.send(SECOND_FUTURE_TOPIC, "Testing topic 2"); - assertThat(awaitLatch(latchContainer.futureCountDownLatch2)).isTrue(); + void shouldRetrySecondTopic() { + kafkaTemplate.send(SECOND_TOPIC, "Testing topic 2"); + assertThat(awaitLatch(latchContainer.countDownLatch2)).isTrue(); assertThat(awaitLatch(latchContainer.customDltCountdownLatch)).isTrue(); } @Test - void shouldRetrySecondMonoTopic() { - kafkaTemplate.send(SECOND_MONO_TOPIC, "Testing topic 2"); - assertThat(awaitLatch(latchContainer.monoCountDownLatch2)).isTrue(); - assertThat(awaitLatch(latchContainer.customDltCountdownLatch)).isTrue(); - } - - - @Test - void shouldRetryThirdFutureTopicWithTimeout(@Autowired KafkaAdmin admin, + void shouldRetryThirdTopicWithTimeout(@Autowired KafkaAdmin admin, @Autowired KafkaListenerEndpointRegistry registry) throws Exception { - kafkaTemplate.send(THIRD_FUTURE_TOPIC, "Testing topic 3"); - assertThat(awaitLatch(latchContainer.futureCountDownLatch3)).isTrue(); - assertThat(awaitLatch(latchContainer.countDownLatchDltOne)).isTrue(); - Map topics = admin.describeTopics(THIRD_FUTURE_TOPIC, THIRD_FUTURE_TOPIC + "-dlt", FOURTH_FUTURE_TOPIC); - assertThat(topics.get(THIRD_FUTURE_TOPIC).partitions()).hasSize(2); - assertThat(topics.get(THIRD_FUTURE_TOPIC + "-dlt").partitions()).hasSize(3); - assertThat(topics.get(FOURTH_FUTURE_TOPIC).partitions()).hasSize(2); - AtomicReference method = new AtomicReference<>(); - ReflectionUtils.doWithMethods(KafkaAdmin.class, m -> { - m.setAccessible(true); - method.set(m); - }, m -> m.getName().equals("newTopics")); - @SuppressWarnings("unchecked") - Collection weededTopics = (Collection) method.get().invoke(admin); - AtomicInteger weeded = new AtomicInteger(); - weededTopics.forEach(topic -> { - if (topic.name().equals(THIRD_FUTURE_TOPIC) || topic.name().equals(FOURTH_FUTURE_TOPIC)) { - assertThat(topic).isExactlyInstanceOf(NewTopic.class); - weeded.incrementAndGet(); - } - }); - assertThat(weeded.get()).isEqualTo(2); - registry.getListenerContainerIds().stream() - .filter(id -> id.startsWith(THIRD_FUTURE_LISTENER_ID)) - .forEach(id -> { - ConcurrentMessageListenerContainer container = - (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); - if (id.equals(THIRD_FUTURE_LISTENER_ID)) { - assertThat(container.getConcurrency()).isEqualTo(2); - } - else { - assertThat(container.getConcurrency()) - .describedAs("Expected %s to have concurrency", id) - .isEqualTo(1); - } - }); - } - - @Test - void shouldRetryThirdMonoTopicWithTimeout(@Autowired KafkaAdmin admin, - @Autowired KafkaListenerEndpointRegistry registry) throws Exception { - kafkaTemplate.send(THIRD_MONO_TOPIC, "Testing topic 3"); - assertThat(awaitLatch(latchContainer.monoCountDownLatch3)).isTrue(); + kafkaTemplate.send(THIRD_TOPIC, "Testing topic 3"); + assertThat(awaitLatch(latchContainer.countDownLatch3)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatchDltOne)).isTrue(); - Map topics = admin.describeTopics(THIRD_MONO_TOPIC, THIRD_MONO_TOPIC + "-dlt", FOURTH_MONO_TOPIC); - assertThat(topics.get(THIRD_MONO_TOPIC).partitions()).hasSize(2); - assertThat(topics.get(THIRD_MONO_TOPIC + "-dlt").partitions()).hasSize(3); - assertThat(topics.get(FOURTH_MONO_TOPIC).partitions()).hasSize(2); + Map topics = admin.describeTopics(THIRD_TOPIC, THIRD_TOPIC + "-dlt", FOURTH_TOPIC); + assertThat(topics.get(THIRD_TOPIC).partitions()).hasSize(2); + assertThat(topics.get(THIRD_TOPIC + "-dlt").partitions()).hasSize(3); + assertThat(topics.get(FOURTH_TOPIC).partitions()).hasSize(2); AtomicReference method = new AtomicReference<>(); - ReflectionUtils.doWithMethods(KafkaAdmin.class, m -> { + org.springframework.util.ReflectionUtils.doWithMethods(KafkaAdmin.class, m -> { m.setAccessible(true); method.set(m); }, m -> m.getName().equals("newTopics")); @@ -256,18 +186,18 @@ void shouldRetryThirdMonoTopicWithTimeout(@Autowired KafkaAdmin admin, Collection weededTopics = (Collection) method.get().invoke(admin); AtomicInteger weeded = new AtomicInteger(); weededTopics.forEach(topic -> { - if (topic.name().equals(THIRD_MONO_TOPIC) || topic.name().equals(FOURTH_MONO_TOPIC)) { + if (topic.name().equals(THIRD_TOPIC) || topic.name().equals(FOURTH_TOPIC)) { assertThat(topic).isExactlyInstanceOf(NewTopic.class); weeded.incrementAndGet(); } }); assertThat(weeded.get()).isEqualTo(2); registry.getListenerContainerIds().stream() - .filter(id -> id.startsWith(THIRD_MONO_LISTENER_ID)) + .filter(id -> id.startsWith("third")) .forEach(id -> { ConcurrentMessageListenerContainer container = (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); - if (id.equals(THIRD_MONO_LISTENER_ID)) { + if (id.equals("thirdTopicId")) { assertThat(container.getConcurrency()).isEqualTo(2); } else { @@ -278,52 +208,88 @@ void shouldRetryThirdMonoTopicWithTimeout(@Autowired KafkaAdmin admin, }); } - @Test - void shouldRetryFourthFutureTopicWithNoDlt() { - kafkaTemplate.send(FOURTH_FUTURE_TOPIC, "Testing topic 4"); - assertThat(awaitLatch(latchContainer.futureCountDownLatch4)).isTrue(); + void shouldRetryFourthTopicWithNoDlt() { + kafkaTemplate.send(FOURTH_TOPIC, "Testing topic 4"); + assertThat(awaitLatch(latchContainer.countDownLatch4)).isTrue(); } @Test - void shouldRetryFourthMonoTopicWithNoDlt() { - kafkaTemplate.send(FOURTH_MONO_TOPIC, "Testing topic 4"); - assertThat(awaitLatch(latchContainer.monoCountDownLatch4)).isTrue(); - } - - - - @Test - void shouldRetryFifthTopicWithTwoListenersAndManualAssignment1(@Autowired FifthFutureTopicListener1 listener1, - @Autowired FifthFutureTopicListener2 listener2) { + void shouldRetryFifthTopicWithTwoListenersAndManualAssignment(@Autowired + FifthTopicListener1 listener1, + @Autowired FifthTopicListener2 listener2) { - kafkaTemplate.send(TWO_LISTENERS_FUTURE_TOPIC, 0, "0", "Testing topic 5 - 0"); - kafkaTemplate.send(TWO_LISTENERS_FUTURE_TOPIC, 1, "0", "Testing topic 5 - 1"); - - assertThat(awaitLatch(latchContainer.countDownLatchDltThree)).isTrue(); + kafkaTemplate.send(TWO_LISTENERS_TOPIC, 0, "0", "Testing topic 5 - 0"); + kafkaTemplate.send(TWO_LISTENERS_TOPIC, 1, "0", "Testing topic 5 - 1"); assertThat(awaitLatch(latchContainer.countDownLatch51)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatch52)).isTrue(); - assertThat(listener1.topics).containsExactly(TWO_LISTENERS_FUTURE_TOPIC, TWO_LISTENERS_FUTURE_TOPIC - + "-listener1-0", TWO_LISTENERS_FUTURE_TOPIC + "-listener1-1", TWO_LISTENERS_FUTURE_TOPIC + "-listener1-2", - TWO_LISTENERS_FUTURE_TOPIC + "-listener1-dlt"); - assertThat(listener2.topics).containsExactly(TWO_LISTENERS_FUTURE_TOPIC, TWO_LISTENERS_FUTURE_TOPIC - + "-listener2-0", TWO_LISTENERS_FUTURE_TOPIC + "-listener2-1", TWO_LISTENERS_FUTURE_TOPIC + "-listener2-2", - TWO_LISTENERS_FUTURE_TOPIC + "-listener2-dlt"); + assertThat(awaitLatch(latchContainer.countDownLatchDltThree)).isTrue(); + assertThat(listener1.topics).containsExactly(TWO_LISTENERS_TOPIC, TWO_LISTENERS_TOPIC + + "-listener1-0", TWO_LISTENERS_TOPIC + "-listener1-1", TWO_LISTENERS_TOPIC + "-listener1-2", + TWO_LISTENERS_TOPIC + "-listener1-dlt"); + assertThat(listener2.topics).containsExactly(TWO_LISTENERS_TOPIC, TWO_LISTENERS_TOPIC + + "-listener2-0", TWO_LISTENERS_TOPIC + "-listener2-1", TWO_LISTENERS_TOPIC + "-listener2-2", + TWO_LISTENERS_TOPIC + "-listener2-dlt"); } + @Test + void shouldRetryManualTopicWithDefaultDlt(@Autowired KafkaListenerEndpointRegistry registry, + @Autowired ConsumerFactory cf) { + kafkaTemplate.send(MANUAL_TOPIC, "Testing topic 6"); + assertThat(awaitLatch(latchContainer.countDownLatch6)).isTrue(); + registry.getListenerContainerIds().stream() + .filter(id -> id.startsWith("manual")) + .forEach(id -> { + ConcurrentMessageListenerContainer container = + (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); + assertThat(container).extracting("commonErrorHandler") + .extracting("seekAfterError", InstanceOfAssertFactories.BOOLEAN) + .isFalse(); + }); + Consumer consumer = cf.createConsumer("manual-dlt", ""); + Set tp = + Set.of(new org.apache.kafka.common.TopicPartition(MANUAL_TOPIC + "-dlt", 0)); + consumer.assign(tp); + try { + await().untilAsserted(() -> { + OffsetAndMetadata offsetAndMetadata = consumer.committed(tp).get(tp.iterator().next()); + assertThat(offsetAndMetadata).isNotNull(); + assertThat(offsetAndMetadata.offset()).isEqualTo(1L); + }); + } + finally { + consumer.close(); + } + } @Test - void shouldGoStraightToDltInFuture() { - kafkaTemplate.send(NOT_RETRYABLE_EXCEPTION_FUTURE_TOPIC, "Testing topic with annotation 1"); - assertThat(awaitLatch(latchContainer.futureCountDownLatchNoRetry)).isTrue(); - assertThat(awaitLatch(latchContainer.countDownLatchDltTwo)).isTrue(); + void shouldFirstReuseRetryTopic(@Autowired + FirstReuseRetryTopicListener listener1, + @Autowired + SecondReuseRetryTopicListener listener2, @Autowired + ThirdReuseRetryTopicListener listener3) { + + kafkaTemplate.send(FIRST_REUSE_RETRY_TOPIC, "Testing reuse topic 1"); + kafkaTemplate.send(SECOND_REUSE_RETRY_TOPIC, "Testing reuse topic 2"); + kafkaTemplate.send(THIRD_REUSE_RETRY_TOPIC, "Testing reuse topic 3"); + assertThat(awaitLatch(latchContainer.countDownLatchReuseOne)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchReuseTwo)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchReuseThree)).isTrue(); + assertThat(listener1.topics).containsExactly(FIRST_REUSE_RETRY_TOPIC, + FIRST_REUSE_RETRY_TOPIC + "-retry"); + assertThat(listener2.topics).containsExactly(SECOND_REUSE_RETRY_TOPIC, + SECOND_REUSE_RETRY_TOPIC + "-retry-30", SECOND_REUSE_RETRY_TOPIC + "-retry-60", + SECOND_REUSE_RETRY_TOPIC + "-retry-100", SECOND_REUSE_RETRY_TOPIC + "-retry-100"); + assertThat(listener3.topics).containsExactly(THIRD_REUSE_RETRY_TOPIC, + THIRD_REUSE_RETRY_TOPIC + "-retry", THIRD_REUSE_RETRY_TOPIC + "-retry", + THIRD_REUSE_RETRY_TOPIC + "-retry", THIRD_REUSE_RETRY_TOPIC + "-retry"); } @Test - void shouldGoStraightToDltInMono() { - kafkaTemplate.send(NOT_RETRYABLE_EXCEPTION_MONO_TOPIC, "Testing topic with annotation 1"); - assertThat(awaitLatch(latchContainer.monoCountDownLatchNoRetry)).isTrue(); + void shouldGoStraightToDlt() { + kafkaTemplate.send(NOT_RETRYABLE_EXCEPTION_TOPIC, "Testing topic with annotation 1"); + assertThat(awaitLatch(latchContainer.countDownLatchNoRetry)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatchDltTwo)).isTrue(); } @@ -338,13 +304,13 @@ private boolean awaitLatch(CountDownLatch latch) { } @KafkaListener( - id = FIRST_FUTURE_LISTENER_ID, - topics = FIRST_FUTURE_TOPIC, + id = "firstTopicId", + topics = FIRST_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, errorHandler = "myCustomErrorHandler", contentTypeConverter = "myCustomMessageConverter", concurrency = "2") - static class FirstFutureTopicListener { + static class FirstTopicListener { @Autowired DestinationTopicContainer topicContainer; @@ -352,11 +318,10 @@ static class FirstFutureTopicListener { @Autowired CountDownLatchContainer container; - @KafkaHandler public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { - container.futureCountDownLatch1.countDown(); + container.countDownLatch1.countDown(); try { Thread.sleep(1); } catch (InterruptedException e) { @@ -365,41 +330,11 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEI throw new RuntimeException("Woooops... in topic " + receivedTopic); }); } - } - - - @KafkaListener( - id = FIRST_MONO_LISTENER_ID, - topics = FIRST_MONO_TOPIC, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class FirstTopicMonoListener { - - @Autowired - DestinationTopicContainer topicContainer; - @Autowired - CountDownLatchContainer container; - - @KafkaHandler - public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - return Mono.fromCallable(() -> { - container.monoCountDownLatch1.countDown(); - try { - Thread.sleep(1); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - throw new RuntimeException("Woooops... in topic " + receivedTopic); - }).then(); - } } - - @KafkaListener(topics = SECOND_FUTURE_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) - static class SecondFutureTopicListener { + @KafkaListener(topics = SECOND_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class SecondTopicListener { @Autowired CountDownLatchContainer container; @@ -407,7 +342,7 @@ static class SecondFutureTopicListener { @KafkaHandler public CompletableFuture listenAgain(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { - container.countDownIfNotKnown(receivedTopic, container.futureCountDownLatch2); + container.countDownIfNotKnown(receivedTopic, container.countDownLatch2); try { Thread.sleep(1); } catch (InterruptedException e) { @@ -418,42 +353,20 @@ public CompletableFuture listenAgain(String message, @Header(KafkaHeaders. } } - - @KafkaListener(topics = SECOND_MONO_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) - static class SecondMonoTopicListener { - - @Autowired - CountDownLatchContainer container; - - @KafkaHandler - public Mono listenAgain(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - return Mono.fromCallable(() -> { - container.countDownIfNotKnown(receivedTopic, container.monoCountDownLatch2); - try { - Thread.sleep(1); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - throw new IllegalStateException("Another woooops... " + receivedTopic); - }).then(); - } - } - - @RetryableTopic( attempts = "${five.attempts}", backoff = @Backoff(delay = 250, maxDelay = 1000, multiplier = 1.5), numPartitions = "#{3}", - timeout = "${missing.property:2000}", + timeout = "${missing.property:100000}", include = MyRetryException.class, kafkaTemplate = "${kafka.template}", topicSuffixingStrategy = TopicSuffixingStrategy.SUFFIX_WITH_INDEX_VALUE, concurrency = "1") @KafkaListener( - id = THIRD_FUTURE_LISTENER_ID, - topics = THIRD_FUTURE_TOPIC, + id = "thirdTopicId", + topics = THIRD_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, concurrency = "2") - static class ThirdFutureTopicListener { + static class ThirdTopicListener { @Autowired CountDownLatchContainer container; @@ -461,7 +374,7 @@ static class ThirdFutureTopicListener { @KafkaHandler public CompletableFuture listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { - container.countDownIfNotKnown(receivedTopic, container.futureCountDownLatch3); + container.countDownIfNotKnown(receivedTopic, container.countDownLatch3); try { Thread.sleep(1); } catch (InterruptedException e) { @@ -477,54 +390,11 @@ public void annotatedDltMethod(Object message) { } } - @Component - @RetryableTopic( - attempts = "${five.attempts}", - backoff = @Backoff(delay = 250, maxDelay = 1000, multiplier = 1.5), - numPartitions = "#{3}", - timeout = "${missing.property:2000}", - include = MyRetryException.class, kafkaTemplate = "${kafka.template}", - topicSuffixingStrategy = TopicSuffixingStrategy.SUFFIX_WITH_INDEX_VALUE, - concurrency = "1") - @KafkaListener( - id = THIRD_MONO_LISTENER_ID, - topics = THIRD_MONO_TOPIC, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - concurrency = "2") - static class ThirdMonoTopicListener { - - @Autowired - CountDownLatchContainer container; - - @KafkaHandler - public Mono listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - return Mono.fromCallable(() -> { - container.countDownIfNotKnown(receivedTopic, container.monoCountDownLatch3); - try { - Thread.sleep(1); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - throw new MyRetryException("Annotated woooops... " + receivedTopic); - }).then(); - } - - @DltHandler - public void annotatedDltMethod(Object message) { - container.countDownLatchDltOne.countDown(); - } - } - - - - @RetryableTopic( - dltStrategy = DltStrategy.NO_DLT, - attempts = "4", - backoff = @Backoff(300), + @RetryableTopic(dltStrategy = DltStrategy.NO_DLT, attempts = "4", backoff = @Backoff(300), sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS, kafkaTemplate = "${kafka.template}") - @KafkaListener(topics = FOURTH_FUTURE_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) - static class FourthFutureTopicListener { + @KafkaListener(topics = FOURTH_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class FourthTopicListener { @Autowired CountDownLatchContainer container; @@ -532,7 +402,7 @@ static class FourthFutureTopicListener { @KafkaHandler public CompletableFuture listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { - container.countDownIfNotKnown(receivedTopic, container.futureCountDownLatch4); + container.countDownIfNotKnown(receivedTopic, container.countDownLatch4); try { Thread.sleep(1); } catch (InterruptedException e) { @@ -548,38 +418,6 @@ public void shouldNotGetHere() { } } - @RetryableTopic( - dltStrategy = DltStrategy.NO_DLT, - attempts = "4", - backoff = @Backoff(300), - sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS, - kafkaTemplate = "${kafka.template}") - @KafkaListener(topics = FOURTH_MONO_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) - static class FourthMonoTopicListener { - - @Autowired - CountDownLatchContainer container; - - @KafkaHandler - public Mono listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - return Mono.fromCallable(() -> { - container.countDownIfNotKnown(receivedTopic, container.monoCountDownLatch4); - try { - Thread.sleep(1); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - throw new IllegalStateException("Another woooops... " + receivedTopic); - }).then(); - } - - @DltHandler - public void shouldNotGetHere() { - fail("Dlt should not be processed!"); - } - } - - static class AbstractFifthTopicListener { final List topics = Collections.synchronizedList(new ArrayList<>()); @@ -597,7 +435,7 @@ public void annotatedDltMethod(ConsumerRecord record) { @RetryableTopic( attempts = "4", - backoff = @Backoff(1), + backoff = @Backoff(250), numPartitions = "2", retryTopicSuffix = "-listener1", dltTopicSuffix = "-listener1-dlt", @@ -606,17 +444,16 @@ public void annotatedDltMethod(ConsumerRecord record) { kafkaTemplate = "${kafka.template}") @KafkaListener( id = "fifthTopicId1", - topicPartitions = {@TopicPartition(topic = TWO_LISTENERS_FUTURE_TOPIC, + topicPartitions = {@org.springframework.kafka.annotation.TopicPartition(topic = TWO_LISTENERS_TOPIC, partitionOffsets = @PartitionOffset(partition = "0", initialOffset = "0"))}, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) - static class FifthFutureTopicListener1 extends AbstractFifthTopicListener { + static class FifthTopicListener1 extends AbstractFifthTopicListener { @KafkaHandler public CompletableFuture listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); - container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); return CompletableFuture.supplyAsync(() -> { - + container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); try { Thread.sleep(1); } catch (InterruptedException e) { @@ -625,11 +462,12 @@ public CompletableFuture listenWithAnnotation(String message, @Header(Kafk throw new RuntimeException("Annotated woooops... " + receivedTopic); }); } + } @RetryableTopic( attempts = "4", - backoff = @Backoff(1), + backoff = @Backoff(250), numPartitions = "2", retryTopicSuffix = "-listener2", dltTopicSuffix = "-listener2-dlt", @@ -638,17 +476,16 @@ public CompletableFuture listenWithAnnotation(String message, @Header(Kafk kafkaTemplate = "${kafka.template}") @KafkaListener( id = "fifthTopicId2", - topicPartitions = {@TopicPartition(topic = TWO_LISTENERS_FUTURE_TOPIC, + topicPartitions = {@org.springframework.kafka.annotation.TopicPartition(topic = TWO_LISTENERS_TOPIC, partitionOffsets = @PartitionOffset(partition = "1", initialOffset = "0"))}, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) - static class FifthFutureTopicListener2 extends AbstractFifthTopicListener { + static class FifthTopicListener2 extends AbstractFifthTopicListener { @KafkaHandler public CompletableFuture listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); - container.countDownIfNotKnown(receivedTopic, container.countDownLatch52); return CompletableFuture.supplyAsync(() -> { - + container.countDownLatch52.countDown(); try { Thread.sleep(1); } catch (InterruptedException e) { @@ -660,57 +497,59 @@ public CompletableFuture listenWithAnnotation2(String message, @Header(Kaf } - - @Component - @RetryableTopic(attempts = "3", numPartitions = "3", exclude = MyDontRetryException.class, - backoff = @Backoff(delay = 50, maxDelay = 100, multiplier = 3), - traversingCauses = "true", kafkaTemplate = "${kafka.template}") - @KafkaListener(topics = NOT_RETRYABLE_EXCEPTION_FUTURE_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) - static class NoRetryFutureTopicListener { + @RetryableTopic( + attempts = "4", + backoff = @Backoff(50), + sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS) + @KafkaListener( + id = "manual", + topics = MANUAL_TOPIC, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class SixthTopicDefaultDLTListener { @Autowired CountDownLatchContainer container; @KafkaHandler - public CompletableFuture listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + public CompletableFuture listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic, + @SuppressWarnings("unused") Acknowledgment ack) { return CompletableFuture.supplyAsync(() -> { - container.countDownIfNotKnown(receivedTopic, container.futureCountDownLatchNoRetry); + container.countDownIfNotKnown(receivedTopic, container.countDownLatch6); try { Thread.sleep(1); } catch (InterruptedException e) { throw new RuntimeException(e); } - throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); + throw new IllegalStateException("Another woooops... " + receivedTopic); }); } - @DltHandler - public void annotatedDltMethod(Object message) { - container.countDownLatchDltTwo.countDown(); - } } - @Component - @RetryableTopic(attempts = "3", numPartitions = "3", exclude = MyDontRetryException.class, + @RetryableTopic( + attempts = "3", + numPartitions = "3", + exclude = MyDontRetryException.class, backoff = @Backoff(delay = 50, maxDelay = 100, multiplier = 3), - traversingCauses = "true", kafkaTemplate = "${kafka.template}") - @KafkaListener(topics = NOT_RETRYABLE_EXCEPTION_MONO_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) - static class NoRetryMonoTopicListener { + traversingCauses = "true", + kafkaTemplate = "${kafka.template}") + @KafkaListener(topics = NOT_RETRYABLE_EXCEPTION_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class NoRetryTopicListener { @Autowired CountDownLatchContainer container; @KafkaHandler - public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - return Mono.fromCallable(() -> { - container.countDownIfNotKnown(receivedTopic, container.monoCountDownLatchNoRetry); + public CompletableFuture listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return CompletableFuture.supplyAsync(() -> { + container.countDownIfNotKnown(receivedTopic, container.countDownLatchNoRetry); try { Thread.sleep(1); } catch (InterruptedException e) { throw new RuntimeException(e); } throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); - }).then(); + }); } @DltHandler @@ -719,57 +558,102 @@ public void annotatedDltMethod(Object message) { } } + @RetryableTopic( + attempts = "2", + backoff = @Backoff(50)) + @KafkaListener( + id = "reuseRetry1", + topics = FIRST_REUSE_RETRY_TOPIC, + containerFactory = "retryTopicListenerContainerFactory") + static class FirstReuseRetryTopicListener { - @Configuration - static class KafkaProducerConfig { + final List topics = Collections.synchronizedList(new ArrayList<>()); @Autowired - EmbeddedKafkaBroker broker; + CountDownLatchContainer container; - @Bean - ProducerFactory producerFactory() { - Map configProps = new HashMap<>(); - configProps.put( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, - this.broker.getBrokersAsString()); - configProps.put( - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - configProps.put( - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - return new DefaultKafkaProducerFactory<>(configProps); + @KafkaHandler + public CompletableFuture listen1(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.topics.add(receivedTopic); + return CompletableFuture.supplyAsync(() -> { + container.countDownLatchReuseOne.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Another woooops... " + receivedTopic); + }); } - @Bean("customKafkaTemplate") - KafkaTemplate kafkaTemplate() { - return new KafkaTemplate<>(producerFactory()); - } + } - @Bean - CountDownLatchContainer latchContainer() { - return new CountDownLatchContainer(); + @RetryableTopic( + attempts = "5", + backoff = @Backoff(delay = 30, maxDelay = 100, multiplier = 2)) + @KafkaListener( + id = "reuseRetry2", + topics = SECOND_REUSE_RETRY_TOPIC, + containerFactory = "retryTopicListenerContainerFactory") + static class SecondReuseRetryTopicListener { + + final List topics = Collections.synchronizedList(new ArrayList<>()); + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public CompletableFuture listen2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.topics.add(receivedTopic); + return CompletableFuture.supplyAsync(() -> { + container.countDownLatchReuseTwo.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Another woooops... " + receivedTopic); + }); } + } - @Component - static class CountDownLatchContainer { + @RetryableTopic(attempts = "5", backoff = @Backoff(delay = 1, maxDelay = 5, multiplier = 1.4)) + @KafkaListener(id = "reuseRetry3", topics = THIRD_REUSE_RETRY_TOPIC, + containerFactory = "retryTopicListenerContainerFactory") + static class ThirdReuseRetryTopicListener { - CountDownLatch futureCountDownLatch1 = new CountDownLatch(5); + final List topics = Collections.synchronizedList(new ArrayList<>()); + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public CompletableFuture listen3(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.topics.add(receivedTopic); + return CompletableFuture.supplyAsync(() -> { + container.countDownLatchReuseThree.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Another woooops... " + receivedTopic); + }); - CountDownLatch monoCountDownLatch1 = new CountDownLatch(5); + } - CountDownLatch futureCountDownLatch2 = new CountDownLatch(3); + } - CountDownLatch monoCountDownLatch2 = new CountDownLatch(3); + static class CountDownLatchContainer { - CountDownLatch futureCountDownLatch3 = new CountDownLatch(3); + CountDownLatch countDownLatch1 = new CountDownLatch(5); - CountDownLatch monoCountDownLatch3 = new CountDownLatch(3); + CountDownLatch countDownLatch2 = new CountDownLatch(3); - CountDownLatch futureCountDownLatch4 = new CountDownLatch(4); + CountDownLatch countDownLatch3 = new CountDownLatch(3); - CountDownLatch monoCountDownLatch4 = new CountDownLatch(4); + CountDownLatch countDownLatch4 = new CountDownLatch(4); CountDownLatch countDownLatch51 = new CountDownLatch(4); @@ -777,9 +661,7 @@ static class CountDownLatchContainer { CountDownLatch countDownLatch6 = new CountDownLatch(4); - CountDownLatch futureCountDownLatchNoRetry = new CountDownLatch(1); - - CountDownLatch monoCountDownLatchNoRetry = new CountDownLatch(1); + CountDownLatch countDownLatchNoRetry = new CountDownLatch(1); CountDownLatch countDownLatchDltOne = new CountDownLatch(1); @@ -811,6 +693,20 @@ private void countDownIfNotKnown(String receivedTopic, CountDownLatch countDownL } } + static class MyCustomDltProcessor { + + @Autowired + KafkaTemplate kafkaTemplate; + + @Autowired + CountDownLatchContainer container; + + public void processDltMessage(Object message) { + container.customDltCountdownLatch.countDown(); + throw new RuntimeException("Dlt Error!"); + } + } + @SuppressWarnings("serial") static class MyRetryException extends RuntimeException { MyRetryException(String msg) { @@ -831,166 +727,144 @@ static class RetryTopicConfigurations extends RetryTopicConfigurationSupport { private static final String DLT_METHOD_NAME = "processDltMessage"; @Bean - FirstFutureTopicListener firstTopicListener() { - return new FirstFutureTopicListener(); - } - - @Bean - FirstTopicMonoListener firstTopicMonoListener() { - return new FirstTopicMonoListener(); + RetryTopicConfiguration firstRetryTopic(KafkaTemplate template) { + return RetryTopicConfigurationBuilder + .newInstance() + .fixedBackOff(50) + .maxAttempts(5) + .concurrency(1) + .useSingleTopicForSameIntervals() + .includeTopic(FIRST_TOPIC) + .doNotRetryOnDltFailure() + .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) + .create(template); } @Bean - SecondFutureTopicListener secondFutureTopicListener() { - return new SecondFutureTopicListener(); + RetryTopicConfiguration secondRetryTopic(KafkaTemplate template) { + return RetryTopicConfigurationBuilder + .newInstance() + .exponentialBackoff(500, 2, 10000) + .retryOn(Arrays.asList(IllegalStateException.class, IllegalAccessException.class)) + .traversingCauses() + .includeTopic(SECOND_TOPIC) + .doNotRetryOnDltFailure() + .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) + .create(template); } @Bean - SecondMonoTopicListener secondMonoTopicListener() { - return new SecondMonoTopicListener(); + FirstTopicListener firstTopicListener() { + return new FirstTopicListener(); } @Bean - ThirdFutureTopicListener thirdFutureTopicListener() { - return new ThirdFutureTopicListener(); + KafkaListenerErrorHandler myCustomErrorHandler( + CountDownLatchContainer container) { + return (message, exception) -> { + container.customErrorHandlerCountdownLatch.countDown(); + throw exception; + }; } @Bean - ThirdMonoTopicListener thirdMonoTopicListener() { - return new ThirdMonoTopicListener(); - } + SmartMessageConverter myCustomMessageConverter( + CountDownLatchContainer container) { + return new CompositeMessageConverter(Collections.singletonList(new GenericMessageConverter())) { - @Bean - FourthFutureTopicListener fourthFutureTopicListener() { - return new FourthFutureTopicListener(); + @Override + public Object fromMessage(Message message, Class targetClass, Object conversionHint) { + container.customMessageConverterCountdownLatch.countDown(); + return super.fromMessage(message, targetClass, conversionHint); + } + }; } @Bean - FourthMonoTopicListener fourthMonoTopicListener() { - return new FourthMonoTopicListener(); + SecondTopicListener secondTopicListener() { + return new SecondTopicListener(); } @Bean - NoRetryFutureTopicListener noRetryFutureTopicListener() { - return new NoRetryFutureTopicListener(); + ThirdTopicListener thirdTopicListener() { + return new ThirdTopicListener(); } @Bean - NoRetryMonoTopicListener noRetryMonoTopicListener() { - return new NoRetryMonoTopicListener(); + FourthTopicListener fourthTopicListener() { + return new FourthTopicListener(); } @Bean - FifthFutureTopicListener1 fifthFutureTopicListener1() { - return new FifthFutureTopicListener1(); + FifthTopicListener1 fifthTopicListener1() { + return new FifthTopicListener1(); } @Bean - FifthFutureTopicListener2 fifthFutureTopicListener2() { - return new FifthFutureTopicListener2(); + FifthTopicListener2 fifthTopicListener2() { + return new FifthTopicListener2(); } @Bean - MyCustomDltProcessor myCustomDltProcessor() { - return new MyCustomDltProcessor(); + SixthTopicDefaultDLTListener manualTopicListener() { + return new SixthTopicDefaultDLTListener(); } @Bean - TaskScheduler taskScheduler() { - return new ThreadPoolTaskScheduler(); + NoRetryTopicListener noRetryTopicListener() { + return new NoRetryTopicListener(); } @Bean - KafkaListenerErrorHandler myCustomErrorHandler( - CountDownLatchContainer container) { - return (message, exception) -> { - container.customErrorHandlerCountdownLatch.countDown(); - throw exception; - }; + FirstReuseRetryTopicListener firstReuseRetryTopicListener() { + return new FirstReuseRetryTopicListener(); } @Bean - SmartMessageConverter myCustomMessageConverter( - CountDownLatchContainer container) { - return new CompositeMessageConverter(Collections.singletonList(new GenericMessageConverter())) { - - @Override - public Object fromMessage(Message message, Class targetClass, Object conversionHint) { - container.customMessageConverterCountdownLatch.countDown(); - return super.fromMessage(message, targetClass, conversionHint); - } - }; + SecondReuseRetryTopicListener secondReuseRetryTopicListener() { + return new SecondReuseRetryTopicListener(); } - @Bean - RetryTopicConfiguration firstRetryTopic(KafkaTemplate template) { - return RetryTopicConfigurationBuilder - .newInstance() - .fixedBackOff(50) - .maxAttempts(5) - .concurrency(1) - .useSingleTopicForSameIntervals() - .includeTopic(FIRST_FUTURE_TOPIC) - .doNotRetryOnDltFailure() - .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) - .create(template); + ThirdReuseRetryTopicListener thirdReuseRetryTopicListener() { + return new ThirdReuseRetryTopicListener(); } @Bean - RetryTopicConfiguration firstRetryMonoTopic(KafkaTemplate template) { - return RetryTopicConfigurationBuilder - .newInstance() - .fixedBackOff(50) - .maxAttempts(5) - .concurrency(1) - .useSingleTopicForSameIntervals() - .includeTopic(FIRST_MONO_TOPIC) - .doNotRetryOnDltFailure() - .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) - .create(template); + CountDownLatchContainer latchContainer() { + return new CountDownLatchContainer(); } @Bean - RetryTopicConfiguration secondRetryFutureTopic(KafkaTemplate template) { - return RetryTopicConfigurationBuilder - .newInstance() - .exponentialBackoff(500, 2, 10000) - .retryOn(Arrays.asList(IllegalStateException.class, IllegalAccessException.class)) - .traversingCauses() - .includeTopic(SECOND_FUTURE_TOPIC) - .doNotRetryOnDltFailure() - .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) - .create(template); + MyCustomDltProcessor myCustomDltProcessor() { + return new MyCustomDltProcessor(); } + } - @Bean - RetryTopicConfiguration secondRetryMonoTopic(KafkaTemplate template) { - return RetryTopicConfigurationBuilder - .newInstance() - .exponentialBackoff(500, 2, 10000) - .retryOn(Arrays.asList(IllegalStateException.class, IllegalAccessException.class)) - .traversingCauses() - .includeTopic(SECOND_MONO_TOPIC) - .doNotRetryOnDltFailure() - .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) - .create(template); - } + @Configuration + static class KafkaProducerConfig { + @Autowired + EmbeddedKafkaBroker broker; @Bean - NewTopics topics() { - - NewTopic thirdFutureTopic = TopicBuilder.name(THIRD_FUTURE_TOPIC).partitions(2).replicas(1).build(); - NewTopic thirdMonoTopic = TopicBuilder.name(THIRD_MONO_TOPIC).partitions(2).replicas(1).build(); - NewTopic fourthFutureTopic = TopicBuilder.name(FOURTH_FUTURE_TOPIC).partitions(2).replicas(1).build(); - NewTopic fourthMonoTopic = TopicBuilder.name(FOURTH_MONO_TOPIC).partitions(2).replicas(1).build(); + ProducerFactory producerFactory() { + Map configProps = new HashMap<>(); + configProps.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.broker.getBrokersAsString()); + configProps.put( + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + configProps.put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + return new DefaultKafkaProducerFactory<>(configProps); + } - return new NewTopics( - thirdFutureTopic, - thirdMonoTopic, - fourthFutureTopic, - fourthMonoTopic); + @Bean("customKafkaTemplate") + KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); } } @@ -1008,15 +882,15 @@ KafkaAdmin kafkaAdmin() { return new KafkaAdmin(configs); } -// @Bean -// NewTopic topic() { -// return TopicBuilder.name(THIRD_TOPIC).partitions(2).replicas(1).build(); -// } -// -// @Bean -// NewTopics topics() { -// return new NewTopics(TopicBuilder.name(FOURTH_TOPIC).partitions(2).replicas(1).build()); -// } + @Bean + NewTopic topic() { + return TopicBuilder.name(THIRD_TOPIC).partitions(2).replicas(1).build(); + } + + @Bean + NewTopics topics() { + return new NewTopics(TopicBuilder.name(FOURTH_TOPIC).partitions(2).replicas(1).build()); + } @Bean ConsumerFactory consumerFactory() { @@ -1072,25 +946,11 @@ ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFa return factory; } -// @Bean -// TaskScheduler sched() { -// return new ThreadPoolTaskScheduler(); -// } + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } } - @Component - static class MyCustomDltProcessor { - - @Autowired - KafkaTemplate kafkaTemplate; - - @Autowired - CountDownLatchContainer container; - - public void processDltMessage(Object message) { - container.customDltCountdownLatch.countDown(); - throw new RuntimeException("Dlt Error!"); - } - } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java new file mode 100644 index 0000000000..6f27b1fea7 --- /dev/null +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java @@ -0,0 +1,958 @@ +/* + * Copyright 2018-2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.kafka.retrytopic; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.awaitility.Awaitility.await; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.annotation.DltHandler; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.annotation.KafkaHandler; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.annotation.PartitionOffset; +import org.springframework.kafka.annotation.RetryableTopic; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.config.KafkaListenerEndpointRegistry; +import org.springframework.kafka.config.TopicBuilder; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaAdmin; +import org.springframework.kafka.core.KafkaAdmin.NewTopics; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; +import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.listener.ContainerProperties.AckMode; +import org.springframework.kafka.listener.KafkaListenerErrorHandler; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.messaging.Message; +import org.springframework.messaging.converter.CompositeMessageConverter; +import org.springframework.messaging.converter.GenericMessageConverter; +import org.springframework.messaging.converter.SmartMessageConverter; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.retry.annotation.Backoff; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; + +import reactor.core.publisher.Mono; + +/** + * @author Sanghyeok An + * @since 3.3.0 + */ + +@SpringJUnitConfig +@DirtiesContext +@EmbeddedKafka(topics = { AsyncMonoFutureRetryTopicClassLevelIntegrationTests.FIRST_TOPIC, + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.SECOND_TOPIC, + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.THIRD_TOPIC, + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.FOURTH_TOPIC, + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.TWO_LISTENERS_TOPIC, + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.MANUAL_TOPIC }) +@TestPropertySource(properties = { "five.attempts=5", "kafka.template=customKafkaTemplate"}) +public class AsyncMonoFutureRetryTopicClassLevelIntegrationTests { + + public final static String FIRST_TOPIC = "myRetryTopic1"; + + public final static String SECOND_TOPIC = "myRetryTopic2"; + + public final static String THIRD_TOPIC = "myRetryTopic3"; + + public final static String FOURTH_TOPIC = "myRetryTopic4"; + + public final static String TWO_LISTENERS_TOPIC = "myRetryTopic5"; + + public final static String MANUAL_TOPIC = "myRetryTopic6"; + + public final static String NOT_RETRYABLE_EXCEPTION_TOPIC = "noRetryTopic"; + + public final static String FIRST_REUSE_RETRY_TOPIC = "reuseRetry1"; + + public final static String SECOND_REUSE_RETRY_TOPIC = "reuseRetry2"; + + public final static String THIRD_REUSE_RETRY_TOPIC = "reuseRetry3"; + + private final static String MAIN_TOPIC_CONTAINER_FACTORY = "kafkaListenerContainerFactory"; + + @Autowired + private KafkaTemplate kafkaTemplate; + + @Autowired + private CountDownLatchContainer latchContainer; + + @Autowired + DestinationTopicContainer topicContainer; + + @Test + void shouldRetryFirstTopic(@Autowired KafkaListenerEndpointRegistry registry) { + kafkaTemplate.send(FIRST_TOPIC, "Testing topic 1"); + assertThat(topicContainer.getNextDestinationTopicFor("firstTopicId", FIRST_TOPIC).getDestinationName()) + .isEqualTo("myRetryTopic1-retry"); + assertThat(awaitLatch(latchContainer.countDownLatch1)).isTrue(); + assertThat(awaitLatch(latchContainer.customDltCountdownLatch)).isTrue(); + assertThat(awaitLatch(latchContainer.customErrorHandlerCountdownLatch)).isTrue(); + assertThat(awaitLatch(latchContainer.customMessageConverterCountdownLatch)).isTrue(); + registry.getListenerContainerIds().stream() + .filter(id -> id.startsWith("first")) + .forEach(id -> { + ConcurrentMessageListenerContainer container = (ConcurrentMessageListenerContainer) registry + .getListenerContainer(id); + if (id.equals("firstTopicId")) { + assertThat(container.getConcurrency()).isEqualTo(2); + } + else { + assertThat(container.getConcurrency()) + .describedAs("Expected %s to have concurrency", id) + .isEqualTo(1); + } + }); + } + + @Test + void shouldRetrySecondTopic() { + kafkaTemplate.send(SECOND_TOPIC, "Testing topic 2"); + assertThat(awaitLatch(latchContainer.countDownLatch2)).isTrue(); + assertThat(awaitLatch(latchContainer.customDltCountdownLatch)).isTrue(); + } + + @Test + void shouldRetryThirdTopicWithTimeout(@Autowired KafkaAdmin admin, + @Autowired KafkaListenerEndpointRegistry registry) throws Exception { + + kafkaTemplate.send(THIRD_TOPIC, "Testing topic 3"); + assertThat(awaitLatch(latchContainer.countDownLatch3)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchDltOne)).isTrue(); + Map topics = admin.describeTopics(THIRD_TOPIC, THIRD_TOPIC + "-dlt", FOURTH_TOPIC); + assertThat(topics.get(THIRD_TOPIC).partitions()).hasSize(2); + assertThat(topics.get(THIRD_TOPIC + "-dlt").partitions()).hasSize(3); + assertThat(topics.get(FOURTH_TOPIC).partitions()).hasSize(2); + AtomicReference method = new AtomicReference<>(); + org.springframework.util.ReflectionUtils.doWithMethods(KafkaAdmin.class, m -> { + m.setAccessible(true); + method.set(m); + }, m -> m.getName().equals("newTopics")); + @SuppressWarnings("unchecked") + Collection weededTopics = (Collection) method.get().invoke(admin); + AtomicInteger weeded = new AtomicInteger(); + weededTopics.forEach(topic -> { + if (topic.name().equals(THIRD_TOPIC) || topic.name().equals(FOURTH_TOPIC)) { + assertThat(topic).isExactlyInstanceOf(NewTopic.class); + weeded.incrementAndGet(); + } + }); + assertThat(weeded.get()).isEqualTo(2); + registry.getListenerContainerIds().stream() + .filter(id -> id.startsWith("third")) + .forEach(id -> { + ConcurrentMessageListenerContainer container = + (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); + if (id.equals("thirdTopicId")) { + assertThat(container.getConcurrency()).isEqualTo(2); + } + else { + assertThat(container.getConcurrency()) + .describedAs("Expected %s to have concurrency", id) + .isEqualTo(1); + } + }); + } + + @Test + void shouldRetryFourthTopicWithNoDlt() { + kafkaTemplate.send(FOURTH_TOPIC, "Testing topic 4"); + assertThat(awaitLatch(latchContainer.countDownLatch4)).isTrue(); + } + + @Test + void shouldRetryFifthTopicWithTwoListenersAndManualAssignment(@Autowired + FifthTopicListener1 listener1, + @Autowired + FifthTopicListener2 listener2) { + + kafkaTemplate.send(TWO_LISTENERS_TOPIC, 0, "0", "Testing topic 5 - 0"); + kafkaTemplate.send(TWO_LISTENERS_TOPIC, 1, "0", "Testing topic 5 - 1"); + assertThat(awaitLatch(latchContainer.countDownLatch51)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatch52)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchDltThree)).isTrue(); + assertThat(listener1.topics).containsExactly(TWO_LISTENERS_TOPIC, TWO_LISTENERS_TOPIC + + "-listener1-0", TWO_LISTENERS_TOPIC + "-listener1-1", TWO_LISTENERS_TOPIC + "-listener1-2", + TWO_LISTENERS_TOPIC + "-listener1-dlt"); + assertThat(listener2.topics).containsExactly(TWO_LISTENERS_TOPIC, TWO_LISTENERS_TOPIC + + "-listener2-0", TWO_LISTENERS_TOPIC + "-listener2-1", TWO_LISTENERS_TOPIC + "-listener2-2", + TWO_LISTENERS_TOPIC + "-listener2-dlt"); + } + + @Test + void shouldRetryManualTopicWithDefaultDlt(@Autowired KafkaListenerEndpointRegistry registry, + @Autowired ConsumerFactory cf) { + + kafkaTemplate.send(MANUAL_TOPIC, "Testing topic 6"); + assertThat(awaitLatch(latchContainer.countDownLatch6)).isTrue(); + registry.getListenerContainerIds().stream() + .filter(id -> id.startsWith("manual")) + .forEach(id -> { + ConcurrentMessageListenerContainer container = + (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); + assertThat(container).extracting("commonErrorHandler") + .extracting("seekAfterError", InstanceOfAssertFactories.BOOLEAN) + .isFalse(); + }); + Consumer consumer = cf.createConsumer("manual-dlt", ""); + Set tp = + Set.of(new org.apache.kafka.common.TopicPartition(MANUAL_TOPIC + "-dlt", 0)); + consumer.assign(tp); + try { + await().untilAsserted(() -> { + OffsetAndMetadata offsetAndMetadata = consumer.committed(tp).get(tp.iterator().next()); + assertThat(offsetAndMetadata).isNotNull(); + assertThat(offsetAndMetadata.offset()).isEqualTo(1L); + }); + } + finally { + consumer.close(); + } + } + + @Test + void shouldFirstReuseRetryTopic(@Autowired + FirstReuseRetryTopicListener listener1, + @Autowired + SecondReuseRetryTopicListener listener2, @Autowired + ThirdReuseRetryTopicListener listener3) { + + kafkaTemplate.send(FIRST_REUSE_RETRY_TOPIC, "Testing reuse topic 1"); + kafkaTemplate.send(SECOND_REUSE_RETRY_TOPIC, "Testing reuse topic 2"); + kafkaTemplate.send(THIRD_REUSE_RETRY_TOPIC, "Testing reuse topic 3"); + assertThat(awaitLatch(latchContainer.countDownLatchReuseOne)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchReuseTwo)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchReuseThree)).isTrue(); + assertThat(listener1.topics).containsExactly(FIRST_REUSE_RETRY_TOPIC, + FIRST_REUSE_RETRY_TOPIC + "-retry"); + assertThat(listener2.topics).containsExactly(SECOND_REUSE_RETRY_TOPIC, + SECOND_REUSE_RETRY_TOPIC + "-retry-30", SECOND_REUSE_RETRY_TOPIC + "-retry-60", + SECOND_REUSE_RETRY_TOPIC + "-retry-100", SECOND_REUSE_RETRY_TOPIC + "-retry-100"); + assertThat(listener3.topics).containsExactly(THIRD_REUSE_RETRY_TOPIC, + THIRD_REUSE_RETRY_TOPIC + "-retry", THIRD_REUSE_RETRY_TOPIC + "-retry", + THIRD_REUSE_RETRY_TOPIC + "-retry", THIRD_REUSE_RETRY_TOPIC + "-retry"); + } + + @Test + void shouldGoStraightToDlt() { + kafkaTemplate.send(NOT_RETRYABLE_EXCEPTION_TOPIC, "Testing topic with annotation 1"); + assertThat(awaitLatch(latchContainer.countDownLatchNoRetry)).isTrue(); + assertThat(awaitLatch(latchContainer.countDownLatchDltTwo)).isTrue(); + } + + private boolean awaitLatch(CountDownLatch latch) { + try { + return latch.await(60, TimeUnit.SECONDS); + } + catch (Exception e) { + fail(e.getMessage()); + throw new RuntimeException(e); + } + } + + @KafkaListener( + id = "firstTopicId", + topics = FIRST_TOPIC, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class FirstTopicListener { + + @Autowired + DestinationTopicContainer topicContainer; + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return Mono.fromCallable(() -> { + container.countDownLatch1.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Woooops... in topic " + receivedTopic); + }).then(); + } + + } + + @KafkaListener(topics = SECOND_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class SecondTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listenAgain(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return Mono.fromCallable(() -> { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch2); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new IllegalStateException("Another woooops... " + receivedTopic); + }).then(); + } + } + + @RetryableTopic( + attempts = "${five.attempts}", + backoff = @Backoff(delay = 250, maxDelay = 1000, multiplier = 1.5), + numPartitions = "#{3}", + timeout = "${missing.property:100000}", + include = MyRetryException.class, kafkaTemplate = "${kafka.template}", + topicSuffixingStrategy = TopicSuffixingStrategy.SUFFIX_WITH_INDEX_VALUE, + concurrency = "1") + @KafkaListener( + id = "thirdTopicId", + topics = THIRD_TOPIC, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + concurrency = "2") + static class ThirdTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return Mono.fromCallable(() -> { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch3); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new MyRetryException("Annotated woooops... " + receivedTopic); + }).then(); + } + + @DltHandler + public void annotatedDltMethod(Object message) { + container.countDownLatchDltOne.countDown(); + } + } + + @RetryableTopic(dltStrategy = DltStrategy.NO_DLT, attempts = "4", backoff = @Backoff(300), + sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS, + kafkaTemplate = "${kafka.template}") + @KafkaListener(topics = FOURTH_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class FourthTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return Mono.fromCallable(() -> { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch4); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new IllegalStateException("Another woooops... " + receivedTopic); + }).then(); + } + + @DltHandler + public void shouldNotGetHere() { + fail("Dlt should not be processed!"); + } + } + + static class AbstractFifthTopicListener { + + final List topics = Collections.synchronizedList(new ArrayList<>()); + + @Autowired + CountDownLatchContainer container; + + @DltHandler + public void annotatedDltMethod(ConsumerRecord record) { + this.topics.add(record.topic()); + container.countDownLatchDltThree.countDown(); + } + + } + + @RetryableTopic( + attempts = "4", + backoff = @Backoff(250), + numPartitions = "2", + retryTopicSuffix = "-listener1", + dltTopicSuffix = "-listener1-dlt", + topicSuffixingStrategy = TopicSuffixingStrategy.SUFFIX_WITH_INDEX_VALUE, + sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS, + kafkaTemplate = "${kafka.template}") + @KafkaListener( + id = "fifthTopicId1", + topicPartitions = {@org.springframework.kafka.annotation.TopicPartition(topic = TWO_LISTENERS_TOPIC, + partitionOffsets = @PartitionOffset(partition = "0", initialOffset = "0"))}, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class FifthTopicListener1 extends + AbstractFifthTopicListener { + + @KafkaHandler + public Mono listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.topics.add(receivedTopic); + return Mono.fromCallable(() -> { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Annotated woooops... " + receivedTopic); + }).then(); + } + + } + + @RetryableTopic( + attempts = "4", + backoff = @Backoff(250), + numPartitions = "2", + retryTopicSuffix = "-listener2", + dltTopicSuffix = "-listener2-dlt", + topicSuffixingStrategy = TopicSuffixingStrategy.SUFFIX_WITH_INDEX_VALUE, + sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS, + kafkaTemplate = "${kafka.template}") + @KafkaListener( + id = "fifthTopicId2", + topicPartitions = {@org.springframework.kafka.annotation.TopicPartition(topic = TWO_LISTENERS_TOPIC, + partitionOffsets = @PartitionOffset(partition = "1", initialOffset = "0"))}, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class FifthTopicListener2 extends + AbstractFifthTopicListener { + + @KafkaHandler + public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.topics.add(receivedTopic); + return Mono.fromCallable(() -> { + container.countDownLatch52.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Annotated woooops... " + receivedTopic); + }).then(); + } + + } + + @RetryableTopic( + attempts = "4", + backoff = @Backoff(50), + sameIntervalTopicReuseStrategy = SameIntervalTopicReuseStrategy.MULTIPLE_TOPICS) + @KafkaListener( + id = "manual", + topics = MANUAL_TOPIC, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class SixthTopicDefaultDLTListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic, + @SuppressWarnings("unused") Acknowledgment ack) { + return Mono.fromCallable(() -> { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch6); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new IllegalStateException("Another woooops... " + receivedTopic); + }).then(); + } + + } + + @RetryableTopic( + attempts = "3", + numPartitions = "3", + exclude = MyDontRetryException.class, + backoff = @Backoff(delay = 50, maxDelay = 100, multiplier = 3), + traversingCauses = "true", + kafkaTemplate = "${kafka.template}") + @KafkaListener(topics = NOT_RETRYABLE_EXCEPTION_TOPIC, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) + static class NoRetryTopicListener { + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + return Mono.fromCallable(() -> { + container.countDownIfNotKnown(receivedTopic, container.countDownLatchNoRetry); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); + }).then(); + } + + @DltHandler + public void annotatedDltMethod(Object message) { + container.countDownLatchDltTwo.countDown(); + } + } + + @RetryableTopic( + attempts = "2", + backoff = @Backoff(50)) + @KafkaListener( + id = "reuseRetry1", + topics = FIRST_REUSE_RETRY_TOPIC, + containerFactory = "retryTopicListenerContainerFactory") + static class FirstReuseRetryTopicListener { + + final List topics = Collections.synchronizedList(new ArrayList<>()); + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listen1(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.topics.add(receivedTopic); + return Mono.fromCallable(() -> { + container.countDownLatchReuseOne.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Another woooops... " + receivedTopic); + }).then(); + } + + } + + @RetryableTopic( + attempts = "5", + backoff = @Backoff(delay = 30, maxDelay = 100, multiplier = 2)) + @KafkaListener( + id = "reuseRetry2", + topics = SECOND_REUSE_RETRY_TOPIC, + containerFactory = "retryTopicListenerContainerFactory") + static class SecondReuseRetryTopicListener { + + final List topics = Collections.synchronizedList(new ArrayList<>()); + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listen2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.topics.add(receivedTopic); + return Mono.fromCallable(() -> { + container.countDownLatchReuseTwo.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Another woooops... " + receivedTopic); + }).then(); + } + + } + + @RetryableTopic(attempts = "5", backoff = @Backoff(delay = 1, maxDelay = 5, multiplier = 1.4)) + @KafkaListener(id = "reuseRetry3", topics = THIRD_REUSE_RETRY_TOPIC, + containerFactory = "retryTopicListenerContainerFactory") + static class ThirdReuseRetryTopicListener { + + final List topics = Collections.synchronizedList(new ArrayList<>()); + + @Autowired + CountDownLatchContainer container; + + @KafkaHandler + public Mono listen3(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.topics.add(receivedTopic); + return Mono.fromCallable(() -> { + container.countDownLatchReuseThree.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Another woooops... " + receivedTopic); + }).then(); + } + + } + + static class CountDownLatchContainer { + + CountDownLatch countDownLatch1 = new CountDownLatch(5); + + CountDownLatch countDownLatch2 = new CountDownLatch(3); + + CountDownLatch countDownLatch3 = new CountDownLatch(3); + + CountDownLatch countDownLatch4 = new CountDownLatch(4); + + CountDownLatch countDownLatch51 = new CountDownLatch(4); + + CountDownLatch countDownLatch52 = new CountDownLatch(4); + + CountDownLatch countDownLatch6 = new CountDownLatch(4); + + CountDownLatch countDownLatchNoRetry = new CountDownLatch(1); + + CountDownLatch countDownLatchDltOne = new CountDownLatch(1); + + CountDownLatch countDownLatchDltTwo = new CountDownLatch(1); + + CountDownLatch countDownLatchDltThree = new CountDownLatch(2); + + CountDownLatch countDownLatchReuseOne = new CountDownLatch(2); + + CountDownLatch countDownLatchReuseTwo = new CountDownLatch(5); + + CountDownLatch countDownLatchReuseThree = new CountDownLatch(5); + + CountDownLatch customDltCountdownLatch = new CountDownLatch(1); + + CountDownLatch customErrorHandlerCountdownLatch = new CountDownLatch(6); + + CountDownLatch customMessageConverterCountdownLatch = new CountDownLatch(6); + + final List knownTopics = new ArrayList<>(); + + private void countDownIfNotKnown(String receivedTopic, CountDownLatch countDownLatch) { + synchronized (knownTopics) { + if (!knownTopics.contains(receivedTopic)) { + knownTopics.add(receivedTopic); + countDownLatch.countDown(); + } + } + } + } + + static class MyCustomDltProcessor { + + @Autowired + KafkaTemplate kafkaTemplate; + + @Autowired + CountDownLatchContainer container; + + public void processDltMessage(Object message) { + container.customDltCountdownLatch.countDown(); + throw new RuntimeException("Dlt Error!"); + } + } + + @SuppressWarnings("serial") + static class MyRetryException extends RuntimeException { + MyRetryException(String msg) { + super(msg); + } + } + + @SuppressWarnings("serial") + static class MyDontRetryException extends RuntimeException { + MyDontRetryException(String msg) { + super(msg); + } + } + + @Configuration + static class RetryTopicConfigurations extends RetryTopicConfigurationSupport { + + private static final String DLT_METHOD_NAME = "processDltMessage"; + + @Bean + RetryTopicConfiguration firstRetryTopic(KafkaTemplate template) { + return RetryTopicConfigurationBuilder + .newInstance() + .fixedBackOff(50) + .maxAttempts(5) + .concurrency(1) + .useSingleTopicForSameIntervals() + .includeTopic(FIRST_TOPIC) + .doNotRetryOnDltFailure() + .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) + .create(template); + } + + @Bean + RetryTopicConfiguration secondRetryTopic(KafkaTemplate template) { + return RetryTopicConfigurationBuilder + .newInstance() + .exponentialBackoff(500, 2, 10000) + .retryOn(Arrays.asList(IllegalStateException.class, IllegalAccessException.class)) + .traversingCauses() + .includeTopic(SECOND_TOPIC) + .doNotRetryOnDltFailure() + .dltHandlerMethod("myCustomDltProcessor", DLT_METHOD_NAME) + .create(template); + } + + @Bean + FirstTopicListener firstTopicListener() { + return new FirstTopicListener(); + } + + @Bean + KafkaListenerErrorHandler myCustomErrorHandler( + CountDownLatchContainer container) { + return (message, exception) -> { + container.customErrorHandlerCountdownLatch.countDown(); + throw exception; + }; + } + + @Bean + SmartMessageConverter myCustomMessageConverter( + CountDownLatchContainer container) { + return new CompositeMessageConverter(Collections.singletonList(new GenericMessageConverter())) { + + @Override + public Object fromMessage(Message message, Class targetClass, Object conversionHint) { + container.customMessageConverterCountdownLatch.countDown(); + return super.fromMessage(message, targetClass, conversionHint); + } + }; + } + + @Bean + SecondTopicListener secondTopicListener() { + return new SecondTopicListener(); + } + + @Bean + ThirdTopicListener thirdTopicListener() { + return new ThirdTopicListener(); + } + + @Bean + FourthTopicListener fourthTopicListener() { + return new FourthTopicListener(); + } + + @Bean + FifthTopicListener1 fifthTopicListener1() { + return new FifthTopicListener1(); + } + + @Bean + FifthTopicListener2 fifthTopicListener2() { + return new FifthTopicListener2(); + } + + @Bean + SixthTopicDefaultDLTListener manualTopicListener() { + return new SixthTopicDefaultDLTListener(); + } + + @Bean + NoRetryTopicListener noRetryTopicListener() { + return new NoRetryTopicListener(); + } + + @Bean + FirstReuseRetryTopicListener firstReuseRetryTopicListener() { + return new FirstReuseRetryTopicListener(); + } + + @Bean + SecondReuseRetryTopicListener secondReuseRetryTopicListener() { + return new SecondReuseRetryTopicListener(); + } + + @Bean + ThirdReuseRetryTopicListener thirdReuseRetryTopicListener() { + return new ThirdReuseRetryTopicListener(); + } + + @Bean + CountDownLatchContainer latchContainer() { + return new CountDownLatchContainer(); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor() { + return new MyCustomDltProcessor(); + } + } + + @Configuration + static class KafkaProducerConfig { + + @Autowired + EmbeddedKafkaBroker broker; + + @Bean + ProducerFactory producerFactory() { + Map configProps = new HashMap<>(); + configProps.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.broker.getBrokersAsString()); + configProps.put( + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + configProps.put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + return new DefaultKafkaProducerFactory<>(configProps); + } + + @Bean("customKafkaTemplate") + KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); + } + } + + @EnableKafka + @Configuration + static class KafkaConsumerConfig { + + @Autowired + EmbeddedKafkaBroker broker; + + @Bean + KafkaAdmin kafkaAdmin() { + Map configs = new HashMap<>(); + configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, this.broker.getBrokersAsString()); + return new KafkaAdmin(configs); + } + + @Bean + NewTopic topic() { + return TopicBuilder.name(THIRD_TOPIC).partitions(2).replicas(1).build(); + } + + @Bean + NewTopics topics() { + return new NewTopics(TopicBuilder.name(FOURTH_TOPIC).partitions(2).replicas(1).build()); + } + + @Bean + ConsumerFactory consumerFactory() { + Map props = new HashMap<>(); + props.put( + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.broker.getBrokersAsString()); + props.put( + ConsumerConfig.GROUP_ID_CONFIG, + "groupId"); + props.put( + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class); + props.put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class); + props.put( + ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + + return new DefaultKafkaConsumerFactory<>(props); + } + + @Bean + ConcurrentKafkaListenerContainerFactory retryTopicListenerContainerFactory( + ConsumerFactory consumerFactory) { + + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + ContainerProperties props = factory.getContainerProperties(); + props.setIdleEventInterval(100L); + props.setPollTimeout(50L); + props.setIdlePartitionEventInterval(100L); + factory.setConsumerFactory(consumerFactory); + factory.setConcurrency(1); + factory.setContainerCustomizer( + container -> container.getContainerProperties().setIdlePartitionEventInterval(100L)); + return factory; + } + + @Bean + ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory( + ConsumerFactory consumerFactory) { + + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(consumerFactory); + factory.setConcurrency(1); + factory.setContainerCustomizer(container -> { + if (container.getListenerId().startsWith("manual")) { + container.getContainerProperties().setAckMode(AckMode.MANUAL); + container.getContainerProperties().setAsyncAcks(true); + } + }); + return factory; + } + + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + + } + +} From 723cc18abab2401fa62486e6a1ff2b964502f2c4 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sun, 6 Oct 2024 15:45:29 +0900 Subject: [PATCH 04/30] Add test cases. --- .../MessagingMessageListenerAdapter.java | 4 - ...RecordMessagingMessageListenerAdapter.java | 6 + ...eRetryTopicClassLevelIntegrationTests.java | 2 +- ...pletableFutureRetryTopicScenarioTests.java | 1284 +++++++++++++++++ .../AsyncMonoRetryTopicScenarioTests.java | 1254 ++++++++++++++++ 5 files changed, 2545 insertions(+), 5 deletions(-) create mode 100644 spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java create mode 100644 spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index fed47f5f56..edc7c1428e 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -911,10 +911,6 @@ public void acknowledge() { } - public void setAsyncRetryCallback(BiConsumer, RuntimeException> asyncRetryCallback) { - this.asyncRetryCallback = asyncRetryCallback; - } - public void putInAsyncFailureQueue(java.util.function.Consumer callbackForAsyncFailureQueue) { this.callbackForAsyncFailureQueue = callbackForAsyncFailureQueue; } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java index 6caa854e45..24f14241c2 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.springframework.kafka.core.FailedRecordTuple; import org.springframework.kafka.listener.AcknowledgingConsumerAwareMessageListener; import org.springframework.kafka.listener.KafkaListenerErrorHandler; import org.springframework.kafka.support.Acknowledgment; @@ -85,4 +86,9 @@ public void onMessage(ConsumerRecord record, @Nullable Acknowledgment ackn invoke(record, acknowledgment, consumer, message); } + @Override + public void setCallbackForAsyncFailureQueue( + java.util.function.Consumer asyncRetryCallback) { + putInAsyncFailureQueue(asyncRetryCallback); + } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java index 45f4d3947c..9564ecf810 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 the original author or authors. + * Copyright 2018-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java new file mode 100644 index 0000000000..6bc7079d99 --- /dev/null +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -0,0 +1,1284 @@ +/* + * Copyright 2018-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.kafka.retrytopic; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.annotation.KafkaHandler; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.config.KafkaListenerEndpointRegistry; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaAdmin; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.listener.ContainerProperties.AckMode; +import org.springframework.kafka.listener.KafkaListenerErrorHandler; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.messaging.Message; +import org.springframework.messaging.converter.CompositeMessageConverter; +import org.springframework.messaging.converter.GenericMessageConverter; +import org.springframework.messaging.converter.SmartMessageConverter; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; + +/** + * @author Sanghyeok An + * @since 3.3.0 + */ + +@SpringJUnitConfig +@DirtiesContext +@EmbeddedKafka +@TestPropertySource(properties = { "five.attempts=5", "kafka.template=customKafkaTemplate"}) +public class AsyncCompletableFutureRetryTopicScenarioTests { + + private final static String MAIN_TOPIC_CONTAINER_FACTORY = "kafkaListenerContainerFactory"; + + public final static String TEST_TOPIC0 = "myRetryTopic0"; + + public final static String TEST_TOPIC1 = "myRetryTopic1"; + + public final static String TEST_TOPIC2 = "myRetryTopic2"; + + public final static String TEST_TOPIC3 = "myRetryTopic3"; + + public final static String TEST_TOPIC4 = "myRetryTopic4"; + + public final static String TEST_TOPIC5 = "myRetryTopic5"; + + public final static String TEST_TOPIC6 = "myRetryTopic6"; + + + @Autowired + private KafkaTemplate kafkaTemplate; + + @Autowired + private CountDownLatchContainer latchContainer; + + @Autowired + DestinationTopicContainer topicContainer; + + @KafkaListener( + id = "0-topicId", + topics = TEST_TOPIC0, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener0 { + + @Autowired + CountDownLatchContainer container; + + private final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch0.countDown(); + container.countDownLatch0.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Woooops... in topic " + receivedTopic); + }); + } + + } + + @Test + void allFailCaseTest(@Autowired KafkaListenerEndpointRegistry registry, + @Autowired TestTopicListener0 zeroTopicListener, + @Autowired MyCustomDltProcessor myCustomDltProcessor0, + @Autowired ThreadPoolTaskExecutor executor) { + // All Fail case. + String shortFailedMsg1 = "0"; + String shortFailedMsg2 = "1"; + String shortFailedMsg3 = "2"; + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("0-topicId", TEST_TOPIC0); + + String expectedRetryTopic = TEST_TOPIC0 + "-retry"; + String[] expectedReceivedMsgs = { + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3, + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3, + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3, + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3, + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3 + }; + String[] expectedReceivedTopics = { + TEST_TOPIC0, + TEST_TOPIC0, + TEST_TOPIC0, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic + }; + String[] expectedDltMsgs = { + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3 + }; + + // When + kafkaTemplate.send(TEST_TOPIC0, shortFailedMsg1); + kafkaTemplate.send(TEST_TOPIC0, shortFailedMsg2); + kafkaTemplate.send(TEST_TOPIC0, shortFailedMsg3); + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch0)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch0)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(TEST_TOPIC0 + "-retry"); + + assertThat(zeroTopicListener.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(zeroTopicListener.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); + assertThat(latchContainer.extraCountDownLatch0.getCount()).isEqualTo(1000 - CountDownLatchContainer.COUNT0); + + assertThat(myCustomDltProcessor0.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch0.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT0); + } + + @KafkaListener( + id = "1-topicId", + topics = TEST_TOPIC1, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener1 { + + @Autowired + CountDownLatchContainer container; + + private final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch1.countDown(); + container.countDownLatch1.countDown(); + try { + Thread.sleep(Integer.parseInt(message)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (message.equals("1")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } + + } + + @Test + void firstShortFailAndLastLongSuccessRetryTest(@Autowired TestTopicListener1 testTopicListener1, + @Autowired MyCustomDltProcessor myCustomDltProcessor1, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + String longSuccessMsg = "3"; + String shortFailedMsg = "1"; + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("1-topicId", TEST_TOPIC1); + + String expectedRetryTopic = TEST_TOPIC1 + "-retry"; + String[] expectedReceivedMsgs = { + shortFailedMsg, + longSuccessMsg, + shortFailedMsg, + shortFailedMsg, + shortFailedMsg, + shortFailedMsg + }; + + String[] expectedReceivedTopics = { + TEST_TOPIC1, + TEST_TOPIC1, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic + }; + + String[] expectedDltMsgs = { + shortFailedMsg + }; + + + // When + kafkaTemplate.send(TEST_TOPIC1, shortFailedMsg); + kafkaTemplate.send(TEST_TOPIC1, longSuccessMsg); + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch1)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch1)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(latchContainer.extraCountDownLatch1.getCount()).isEqualTo(1000 - CountDownLatchContainer.COUNT1); + assertThat(testTopicListener1.receivedMsgs).containsExactly(expectedReceivedMsgs); + assertThat(testTopicListener1.receivedTopics).containsExactly(expectedReceivedTopics); + + assertThat(myCustomDltProcessor1.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch1.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT1); + } + + @KafkaListener( + id = "2-topicId", + topics = TEST_TOPIC2, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener2 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch2.countDown(); + container.countDownLatch2.countDown(); + long count = container.countDownLatch2.getCount(); + try { + Thread.sleep(Integer.parseInt(message)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + if (message.equals("1")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } + } + + @Test + void firstLongSuccessAndLastShortFailed(@Autowired TestTopicListener2 zero2TopicListener, + @Autowired MyCustomDltProcessor myCustomDltProcessor2, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + String shortFailedMsg = "1"; + String longSuccessMsg = "3"; + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("2-topicId", TEST_TOPIC2); + + String expectedRetryTopic = TEST_TOPIC2 + "-retry"; + String[] expectedReceivedMsgs = { + longSuccessMsg, + shortFailedMsg, + shortFailedMsg, + shortFailedMsg, + shortFailedMsg, + shortFailedMsg + }; + + String[] expectedReceivedTopics = { + TEST_TOPIC2, + TEST_TOPIC2, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic + }; + + String[] expectedDltMsgs = { + shortFailedMsg + }; + + // When + kafkaTemplate.send(TEST_TOPIC2, longSuccessMsg); + kafkaTemplate.send(TEST_TOPIC2, shortFailedMsg); + + // Then + + assertThat(awaitLatch(latchContainer.countDownLatch2)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch2)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(zero2TopicListener.receivedMsgs).containsExactly(expectedReceivedMsgs); + assertThat(zero2TopicListener.receivedTopics).containsExactly(expectedReceivedTopics); + + assertThat(myCustomDltProcessor2.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch2.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT2); + } + + + @KafkaListener( + id = "3-topicId", + topics = TEST_TOPIC3, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener3 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + public static final String LONG_FAIL_MSG = "5000"; + public static final String SHORT_SUCCESS_MSG = "1"; + + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch3.countDown(); + container.countDownLatch3.countDown(); + long count = container.countDownLatch3.getCount(); + try { + Thread.sleep(Integer.parseInt(message)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + if (message.equals(LONG_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } + } + + + @Test + void longFailMsgTwiceThenShortSucessMsgThird(@Autowired TestTopicListener3 testTopicListener3, + @Autowired MyCustomDltProcessor myCustomDltProcessor3, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("3-topicId", TEST_TOPIC3); + + String expectedRetryTopic = TEST_TOPIC3 + "-retry"; + String[] expectedReceivedMsgs = { + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.SHORT_SUCCESS_MSG, + TestTopicListener3.SHORT_SUCCESS_MSG, + TestTopicListener3.SHORT_SUCCESS_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + }; + + String[] expectedReceivedTopics = { + TEST_TOPIC3, + TEST_TOPIC3, + TEST_TOPIC3, + TEST_TOPIC3, + TEST_TOPIC3, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic + }; + + String[] expectedDltMsgs = { + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + }; + + // When + kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.LONG_FAIL_MSG); + kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.LONG_FAIL_MSG); + kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.SHORT_SUCCESS_MSG); + kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.SHORT_SUCCESS_MSG); + kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.SHORT_SUCCESS_MSG); + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch3)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch3)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(testTopicListener3.receivedMsgs).containsExactly(expectedReceivedMsgs); + assertThat(testTopicListener3.receivedTopics).containsExactly(expectedReceivedTopics); + + assertThat(myCustomDltProcessor3.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch3.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT3); + } + + @KafkaListener( + id = "4-TopicId", + topics = TEST_TOPIC4, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener4 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + public static final String LONG_SUCCESS_MSG = "5000"; + public static final String SHORT_FAIL_MSG = "1"; + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch4.countDown(); + container.countDownLatch4.countDown(); + try { + Thread.sleep(Integer.parseInt(message)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + if (message.equals(SHORT_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } + } + + + @Test + void longSuccessMsgTwiceThenShortFailMsgTwice(@Autowired TestTopicListener4 topicListener4, + @Autowired MyCustomDltProcessor myCustomDltProcessor4, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("4-TopicId", TEST_TOPIC4); + + String expectedRetryTopic = TEST_TOPIC4 + "-retry"; + String[] expectedReceivedMsgs = { + TestTopicListener4.LONG_SUCCESS_MSG, + TestTopicListener4.LONG_SUCCESS_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + }; + + String[] expectedReceivedTopics = { + TEST_TOPIC4, + TEST_TOPIC4, + TEST_TOPIC4, + TEST_TOPIC4, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + }; + + String[] expectedDltMsgs = { + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + }; + + // When + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.LONG_SUCCESS_MSG); + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.LONG_SUCCESS_MSG); + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.SHORT_FAIL_MSG); + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.SHORT_FAIL_MSG); + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch4)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch4)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(topicListener4.receivedMsgs).containsExactly(expectedReceivedMsgs); + assertThat(topicListener4.receivedTopics).containsExactly(expectedReceivedTopics); + + assertThat(myCustomDltProcessor4.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch4.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT4); + } + + + + @KafkaListener( + id = "5-TopicId", + topics = TEST_TOPIC5, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener5 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + public static final String LONG_SUCCESS_MSG = "5000"; + public static final String SHORT_FAIL_MSG = "1"; + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch5.countDown(); + container.countDownLatch5.countDown(); + try { + Thread.sleep(Integer.parseInt(message)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + if (message.startsWith(SHORT_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } + } + + + @Test + void oneLongSuccessMsgBetweenHunderedShortFailMsg(@Autowired TestTopicListener5 topicListener5, + @Autowired MyCustomDltProcessor myCustomDltProcessor5, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("5-TopicId", TEST_TOPIC5); + + String expectedRetryTopic = TEST_TOPIC5 + "-retry"; + + String[] expectedReceivedMsgs = new String[501]; + for (int i = 0; i < 500; i++) { + expectedReceivedMsgs[i] = TestTopicListener5.SHORT_FAIL_MSG; + } + expectedReceivedMsgs[500] = TestTopicListener5.LONG_SUCCESS_MSG; + + + String[] expectedReceivedTopics = new String[501]; + for (int i = 0; i < 100; i++) { + expectedReceivedTopics[i] = TEST_TOPIC5; + } + for (int i = 100; i < 500; i++) { + expectedReceivedTopics[i] = expectedRetryTopic; + } + expectedReceivedTopics[500] = TEST_TOPIC5; + + + String[] expectedDltMsgs = new String[100]; + for (int i = 0; i < 100; i++) { + expectedDltMsgs[i] = TestTopicListener5.SHORT_FAIL_MSG; + } + + // When + for (int i = 0; i < 100; i++) { + kafkaTemplate.send(TEST_TOPIC5, TestTopicListener5.SHORT_FAIL_MSG); + if (i == 50) { + kafkaTemplate.send(TEST_TOPIC5, TestTopicListener5.LONG_SUCCESS_MSG); + } + } + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch5)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch5)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(topicListener5.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(topicListener5.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); + + assertThat(myCustomDltProcessor5.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch5.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT5); + } + + + @KafkaListener( + id = "6-TopicId", + topics = TEST_TOPIC6, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener6 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + public static final String SUCCESS_SUFFIX = ",s"; + public static final String FAIL_SUFFIX = ",f"; + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch6.countDown(); + container.countDownLatch6.countDown(); + + String[] split = message.split(","); + String sleepAWhile = split[0]; + String failOrSuccess = split[1]; + + try { + Thread.sleep(Integer.parseInt(sleepAWhile)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + if (failOrSuccess.equals("f")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } + } + + + @Test + void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime(@Autowired TestTopicListener6 topicListener6, + @Autowired MyCustomDltProcessor myCustomDltProcessor6, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("6-TopicId", TEST_TOPIC6); + + String expectedRetryTopic = TEST_TOPIC6 + "-retry"; + + Random random = new Random(); + ConcurrentLinkedQueue q = new ConcurrentLinkedQueue<>(); + + for (int i = 0; i < 50; i++) { + int randomSleepAWhile = random.nextInt(1, 100); + String msg = String.valueOf(randomSleepAWhile) + TestTopicListener6.SUCCESS_SUFFIX; + q.add(msg); + } + + for (int i = 0; i < 50; i++) { + int randomSleepAWhile = random.nextInt(1, 100); + String msg = String.valueOf(randomSleepAWhile) + TestTopicListener6.FAIL_SUFFIX; + q.add(msg); + } + + int expectedSuccessMsgCount = 50; + int expectedFailedMsgCount = 250; + + int expectedReceivedOriginalTopicCount = 100; + int expectedReceivedRetryTopicCount = 200; + int expectedReceivedDltMsgCount = 50; + + + // When + while (!q.isEmpty()) { + String successOrFailMsg = q.poll(); + kafkaTemplate.send(TEST_TOPIC6, successOrFailMsg); + } + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch6)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch6)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + + long actualReceivedSuccessMsgCount = topicListener6.receivedMsgs.stream() + .map(s -> s.split(",")[1]) + .filter(m -> (',' + m).equals(TestTopicListener6.SUCCESS_SUFFIX)) + .count(); + + long actualReceivedFailedMsgCount = topicListener6.receivedMsgs.stream() + .map(s -> s.split(",")[1]) + .filter(m -> (',' + m).equals( + TestTopicListener6.FAIL_SUFFIX)) + .count(); + + + long actualReceivedOriginalTopicMsgCount = topicListener6.receivedTopics.stream() + .filter(topic -> topic.equals(TEST_TOPIC6)) + .count(); + + long actualReceivedRetryTopicMsgCount = topicListener6.receivedTopics.stream() + .filter(topic -> topic.equals(expectedRetryTopic)) + .count(); + + assertThat(actualReceivedSuccessMsgCount).isEqualTo(expectedSuccessMsgCount); + assertThat(actualReceivedFailedMsgCount).isEqualTo(expectedFailedMsgCount); + assertThat(actualReceivedOriginalTopicMsgCount).isEqualTo(expectedReceivedOriginalTopicCount); + assertThat(actualReceivedRetryTopicMsgCount).isEqualTo(expectedReceivedRetryTopicCount); + + assertThat(myCustomDltProcessor6.receivedMsg.size()).isEqualTo(expectedReceivedDltMsgCount); + assertThat(latchContainer.extraDltCountdownLatch6.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT6); + } + + + + static class CountDownLatchContainer { + + static int COUNT0 = 15; + + static int DLT_COUNT0 = 3; + + CountDownLatch countDownLatch0 = new CountDownLatch(COUNT0); + + CountDownLatch extraCountDownLatch0 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch0 = new CountDownLatch(DLT_COUNT0); + + CountDownLatch extraDltCountdownLatch0 = new CountDownLatch(1000); + + static int COUNT1 = 6; + + static int DLT_COUNT1 = 1; + + CountDownLatch countDownLatch1 = new CountDownLatch(COUNT1); + + CountDownLatch extraCountDownLatch1 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch1 = new CountDownLatch(DLT_COUNT1); + + CountDownLatch extraDltCountdownLatch1 = new CountDownLatch(1000); + + static int COUNT2 = 6; + + static int DLT_COUNT2 = 1; + + CountDownLatch countDownLatch2 = new CountDownLatch(COUNT2); + + CountDownLatch extraCountDownLatch2 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch2 = new CountDownLatch(DLT_COUNT2); + + CountDownLatch extraDltCountdownLatch2 = new CountDownLatch(1000); + + static int COUNT3 = 13; + + static int DLT_COUNT3 = 2; + + CountDownLatch countDownLatch3 = new CountDownLatch(COUNT3); + + CountDownLatch extraCountDownLatch3 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch3 = new CountDownLatch(DLT_COUNT3); + + CountDownLatch extraDltCountdownLatch3 = new CountDownLatch(1000); + + + static int COUNT4 = 12; + + static int DLT_COUNT4 = 2; + + CountDownLatch countDownLatch4 = new CountDownLatch(COUNT4); + + CountDownLatch extraCountDownLatch4 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch4 = new CountDownLatch(DLT_COUNT4); + + CountDownLatch extraDltCountdownLatch4 = new CountDownLatch(1000); + + + static int COUNT5 = 501; + + static int DLT_COUNT5 = 100; + + CountDownLatch countDownLatch5 = new CountDownLatch(COUNT5); + + CountDownLatch extraCountDownLatch5 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch5 = new CountDownLatch(DLT_COUNT5); + + CountDownLatch extraDltCountdownLatch5 = new CountDownLatch(1000); + + + static int COUNT6 = 250; + + static int DLT_COUNT6 = 50; + + CountDownLatch countDownLatch6 = new CountDownLatch(COUNT6); + + CountDownLatch extraCountDownLatch6 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch6 = new CountDownLatch(DLT_COUNT6); + + CountDownLatch extraDltCountdownLatch6 = new CountDownLatch(1000); + + CountDownLatch customErrorHandlerCountdownLatch = new CountDownLatch(6); + + CountDownLatch customMessageConverterCountdownLatch = new CountDownLatch(6); + + } + + static class MyCustomDltProcessor { + + final List receivedMsg = new ArrayList<>(); + + public MyCustomDltProcessor(KafkaTemplate kafkaTemplate, + CountDownLatch latch, + CountDownLatch extraLatch) { + this.kafkaTemplate = kafkaTemplate; + this.latch = latch; + this.extraLatch = extraLatch; + } + + private final KafkaTemplate kafkaTemplate; + + private final CountDownLatch latch; + + private final CountDownLatch extraLatch; + + public void processDltMessage(String message) { + this.receivedMsg.add(message); + latch.countDown(); + extraLatch.countDown(); + } + } + + @SuppressWarnings("serial") + static class MyRetryException extends RuntimeException { + MyRetryException(String msg) { + super(msg); + } + } + + @SuppressWarnings("serial") + static class MyDontRetryException extends RuntimeException { + MyDontRetryException(String msg) { + super(msg); + } + } + + @Configuration + static class RetryTopicConfigurations extends RetryTopicConfigurationSupport { + + private static final String DLT_METHOD_NAME = "processDltMessage"; + + static RetryTopicConfiguration createRetryTopicConfiguration(KafkaTemplate template, + String topicName, + String dltBeanName) { + return RetryTopicConfigurationBuilder + .newInstance() + .fixedBackOff(50) + .maxAttempts(5) + .concurrency(1) + .useSingleTopicForSameIntervals() + .includeTopic(topicName) + .doNotRetryOnDltFailure() + .dltHandlerMethod(dltBeanName, DLT_METHOD_NAME) + .create(template); + } + + @Bean + RetryTopicConfiguration testRetryTopic0(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC0, + "myCustomDltProcessor0"); + } + + @Bean + RetryTopicConfiguration testRetryTopic1(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC1, + "myCustomDltProcessor1"); + } + + @Bean + RetryTopicConfiguration testRetryTopic2(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC2, + "myCustomDltProcessor2"); + } + + @Bean + RetryTopicConfiguration testRetryTopic3(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC3, + "myCustomDltProcessor3"); + } + + @Bean + RetryTopicConfiguration testRetryTopic4(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC4, + "myCustomDltProcessor4"); + } + + @Bean + RetryTopicConfiguration testRetryTopic5(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC5, + "myCustomDltProcessor5"); + } + + @Bean + RetryTopicConfiguration testRetryTopic6(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC6, + "myCustomDltProcessor6"); + } + + + @Bean + KafkaListenerErrorHandler myCustomErrorHandler( + CountDownLatchContainer container) { + return (message, exception) -> { + container.customErrorHandlerCountdownLatch.countDown(); + throw exception; + }; + } + + @Bean + SmartMessageConverter myCustomMessageConverter( + CountDownLatchContainer container) { + return new CompositeMessageConverter(Collections.singletonList(new GenericMessageConverter())) { + + @Override + public Object fromMessage(Message message, Class targetClass, Object conversionHint) { + container.customMessageConverterCountdownLatch.countDown(); + return super.fromMessage(message, targetClass, conversionHint); + } + }; + } + + @Bean + CountDownLatchContainer latchContainer() { + return new CountDownLatchContainer(); + } + + @Bean + TestTopicListener0 testTopicListener0() { + return new TestTopicListener0(); + } + + @Bean + TestTopicListener1 testTopicListener1() { + return new TestTopicListener1(); + } + + @Bean + TestTopicListener2 testTopicListener2() { + return new TestTopicListener2(); + } + + @Bean + TestTopicListener3 testTopicListener3() { + return new TestTopicListener3(); + } + + @Bean + TestTopicListener4 testTopicListener4() { + return new TestTopicListener4(); + } + + @Bean + TestTopicListener5 testTopicListener5() { + return new TestTopicListener5(); + } + + @Bean + TestTopicListener6 testTopicListener6() { + return new TestTopicListener6(); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor0(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch0, + latchContainer.extraDltCountdownLatch0); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor1(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch1, + latchContainer.extraDltCountdownLatch1); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor2(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch2, + latchContainer.extraDltCountdownLatch2); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor3(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch3, + latchContainer.extraDltCountdownLatch3); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor4(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch4, + latchContainer.extraDltCountdownLatch4); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor5(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch5, + latchContainer.extraDltCountdownLatch5); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor6(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch6, + latchContainer.extraDltCountdownLatch6); + } + + } + + @Configuration + static class KafkaProducerConfig { + + @Autowired + EmbeddedKafkaBroker broker; + + @Bean + ProducerFactory producerFactory() { + Map configProps = new HashMap<>(); + configProps.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.broker.getBrokersAsString()); + configProps.put( + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + configProps.put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + return new DefaultKafkaProducerFactory<>(configProps); + } + + @Bean("customKafkaTemplate") + KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); + } + } + + @EnableKafka + @Configuration + static class KafkaConsumerConfig { + + @Autowired + EmbeddedKafkaBroker broker; + + @Bean + KafkaAdmin kafkaAdmin() { + Map configs = new HashMap<>(); + configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, this.broker.getBrokersAsString()); + return new KafkaAdmin(configs); + } + + + @Bean + ConsumerFactory consumerFactory() { + Map props = new HashMap<>(); + props.put( + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.broker.getBrokersAsString()); + props.put( + ConsumerConfig.GROUP_ID_CONFIG, + "groupId"); + props.put( + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class); + props.put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class); + props.put( + ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + + return new DefaultKafkaConsumerFactory<>(props); + } + + @Bean + ConcurrentKafkaListenerContainerFactory retryTopicListenerContainerFactory( + ConsumerFactory consumerFactory) { + + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + ContainerProperties props = factory.getContainerProperties(); + props.setIdleEventInterval(100L); + props.setPollTimeout(50L); + props.setIdlePartitionEventInterval(100L); + factory.setConsumerFactory(consumerFactory); + factory.setConcurrency(1); + factory.setContainerCustomizer( + container -> container.getContainerProperties().setIdlePartitionEventInterval(100L)); + return factory; + } + + @Bean + ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory( + ConsumerFactory consumerFactory) { + + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(consumerFactory); + factory.setConcurrency(1); + factory.setContainerCustomizer(container -> { + if (container.getListenerId().startsWith("manual")) { + container.getContainerProperties().setAckMode(AckMode.MANUAL); + container.getContainerProperties().setAsyncAcks(true); + } + }); + return factory; + } + + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + + @Bean + ThreadPoolTaskExecutor threadPoolTaskExecutor() { + return new ThreadPoolTaskExecutor(); + } + + } + + private boolean awaitLatch(CountDownLatch latch) { + try { + return latch.await(60, TimeUnit.SECONDS); + } + catch (Exception e) { + fail(e.getMessage()); + throw new RuntimeException(e); + } + } + + private void waitAWhile(ThreadPoolTaskExecutor executor, int sleep) { + CountDownLatch countDownLatch = new CountDownLatch(1); + + Thread thread = new Thread(() -> { + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + countDownLatch.countDown(); + }); + + executor.execute(thread); + assertThat(awaitLatch(countDownLatch)).isTrue(); + } + +} diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java new file mode 100644 index 0000000000..62623e50ee --- /dev/null +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -0,0 +1,1254 @@ +/* + * Copyright 2018-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.kafka.retrytopic; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.annotation.KafkaHandler; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.config.KafkaListenerEndpointRegistry; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaAdmin; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.listener.ContainerProperties.AckMode; +import org.springframework.kafka.listener.KafkaListenerErrorHandler; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.kafka.test.EmbeddedKafkaBroker; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.messaging.Message; +import org.springframework.messaging.converter.CompositeMessageConverter; +import org.springframework.messaging.converter.GenericMessageConverter; +import org.springframework.messaging.converter.SmartMessageConverter; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; + +import reactor.core.publisher.Mono; + +/** + * @author Sanghyeok An + * @since 3.3.0 + */ + +@SpringJUnitConfig +@DirtiesContext +@EmbeddedKafka +@TestPropertySource(properties = { "five.attempts=5", "kafka.template=customKafkaTemplate"}) +public class AsyncMonoRetryTopicScenarioTests { + + private final static String MAIN_TOPIC_CONTAINER_FACTORY = "kafkaListenerContainerFactory"; + + public final static String TEST_TOPIC0 = "myRetryTopic0"; + + public final static String TEST_TOPIC1 = "myRetryTopic1"; + + public final static String TEST_TOPIC2 = "myRetryTopic2"; + + public final static String TEST_TOPIC3 = "myRetryTopic3"; + + public final static String TEST_TOPIC4 = "myRetryTopic4"; + + public final static String TEST_TOPIC5 = "myRetryTopic5"; + + public final static String TEST_TOPIC6 = "myRetryTopic6"; + + + @Autowired + private KafkaTemplate kafkaTemplate; + + @Autowired + private CountDownLatchContainer latchContainer; + + @Autowired + DestinationTopicContainer topicContainer; + + @KafkaListener( + id = "0-topicId", + topics = TEST_TOPIC0, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener0 { + + @Autowired + CountDownLatchContainer container; + + private final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + return Mono.fromCallable(() -> { + container.extraCountDownLatch0.countDown(); + container.countDownLatch0.countDown(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Woooops... in topic " + receivedTopic); + }).then(); + } + + } + + @Test + void allFailCaseTest(@Autowired TestTopicListener0 zeroTopicListener, + @Autowired MyCustomDltProcessor myCustomDltProcessor0, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + String shortFailedMsg1 = "0"; + String shortFailedMsg2 = "1"; + String shortFailedMsg3 = "2"; + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("0-topicId", TEST_TOPIC0); + + String expectedRetryTopic = TEST_TOPIC0 + "-retry"; + String[] expectedReceivedMsgs = { + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3, + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3, + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3, + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3, + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3 + }; + String[] expectedReceivedTopics = { + TEST_TOPIC0, + TEST_TOPIC0, + TEST_TOPIC0, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic + }; + String[] expectedDltMsgs = { + shortFailedMsg1, + shortFailedMsg2, + shortFailedMsg3 + }; + + // When + kafkaTemplate.send(TEST_TOPIC0, shortFailedMsg1); + kafkaTemplate.send(TEST_TOPIC0, shortFailedMsg2); + kafkaTemplate.send(TEST_TOPIC0, shortFailedMsg3); + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch0)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch0)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(TEST_TOPIC0 + "-retry"); + + assertThat(zeroTopicListener.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(zeroTopicListener.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); + assertThat(latchContainer.extraCountDownLatch0.getCount()).isEqualTo(1000 - CountDownLatchContainer.COUNT0); + + assertThat(myCustomDltProcessor0.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch0.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT0); + } + + @KafkaListener( + id = "1-topicId", + topics = TEST_TOPIC1, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener1 { + + @Autowired + CountDownLatchContainer container; + + private final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch1.countDown(); + container.countDownLatch1.countDown(); + return Mono.delay(Duration.ofMillis(Integer.parseInt(message))) // Thread.sleep 대체 + .flatMap(delay -> { + if ("1".equals(message)) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + }); + } + } + + @Test + void firstShortFailAndLastLongSuccessRetryTest(@Autowired TestTopicListener1 testTopicListener1, + @Autowired MyCustomDltProcessor myCustomDltProcessor1, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + String longSuccessMsg = "3"; + String shortFailedMsg = "1"; + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("1-topicId", TEST_TOPIC1); + + String expectedRetryTopic = TEST_TOPIC1 + "-retry"; + String[] expectedReceivedMsgs = { + shortFailedMsg, + longSuccessMsg, + shortFailedMsg, + shortFailedMsg, + shortFailedMsg, + shortFailedMsg + }; + + String[] expectedReceivedTopics = { + TEST_TOPIC1, + TEST_TOPIC1, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic + }; + + String[] expectedDltMsgs = { + shortFailedMsg + }; + + // When + kafkaTemplate.send(TEST_TOPIC1, shortFailedMsg); + kafkaTemplate.send(TEST_TOPIC1, longSuccessMsg); + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch1)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch1)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(latchContainer.extraCountDownLatch1.getCount()).isEqualTo(1000 - CountDownLatchContainer.COUNT1); + assertThat(testTopicListener1.receivedMsgs).containsExactly(expectedReceivedMsgs); + assertThat(testTopicListener1.receivedTopics).containsExactly(expectedReceivedTopics); + + assertThat(myCustomDltProcessor1.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch1.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT1); + } + + @KafkaListener( + id = "2-topicId", + topics = TEST_TOPIC2, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener2 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch2.countDown(); + container.countDownLatch2.countDown(); + return Mono.delay(Duration.ofMillis(Integer.parseInt(message))) // Thread.sleep 대체 + .flatMap(delay -> { + if ("1".equals(message)) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + }); + } + } + + @Test + void firstLongSuccessAndLastShortFailed(@Autowired TestTopicListener2 zero2TopicListener, + @Autowired MyCustomDltProcessor myCustomDltProcessor2, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + String shortFailedMsg = "1"; + String longSuccessMsg = "3"; + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("2-topicId", TEST_TOPIC2); + + String expectedRetryTopic = TEST_TOPIC2 + "-retry"; + String[] expectedReceivedMsgs = { + longSuccessMsg, + shortFailedMsg, + shortFailedMsg, + shortFailedMsg, + shortFailedMsg, + shortFailedMsg + }; + + String[] expectedReceivedTopics = { + TEST_TOPIC2, + TEST_TOPIC2, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic + }; + + String[] expectedDltMsgs = { + shortFailedMsg + }; + + // When + kafkaTemplate.send(TEST_TOPIC2, longSuccessMsg); + kafkaTemplate.send(TEST_TOPIC2, shortFailedMsg); + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch2)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch2)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(zero2TopicListener.receivedMsgs).containsExactly(expectedReceivedMsgs); + assertThat(zero2TopicListener.receivedTopics).containsExactly(expectedReceivedTopics); + + assertThat(myCustomDltProcessor2.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch2.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT2); + } + + + @KafkaListener( + id = "3-topicId", + topics = TEST_TOPIC3, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener3 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + public static final String LONG_FAIL_MSG = "5000"; + public static final String SHORT_SUCCESS_MSG = "1"; + + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch3.countDown(); + container.countDownLatch3.countDown(); + return Mono.delay(Duration.ofMillis(Integer.parseInt(message))) // Thread.sleep 대체 + .flatMap(delay -> { + if (message.equals(LONG_FAIL_MSG)) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + }); + } + } + + + @Test + void longFailMsgTwiceThenShortSuccessMsgThird(@Autowired TestTopicListener3 testTopicListener3, + @Autowired MyCustomDltProcessor myCustomDltProcessor3, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("3-topicId", TEST_TOPIC3); + + String expectedRetryTopic = TEST_TOPIC3 + "-retry"; + String[] expectedReceivedMsgs = { + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.SHORT_SUCCESS_MSG, + TestTopicListener3.SHORT_SUCCESS_MSG, + TestTopicListener3.SHORT_SUCCESS_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + }; + + String[] expectedReceivedTopics = { + TEST_TOPIC3, + TEST_TOPIC3, + TEST_TOPIC3, + TEST_TOPIC3, + TEST_TOPIC3, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic + }; + + String[] expectedDltMsgs = { + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + }; + + // When + kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.LONG_FAIL_MSG); + kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.LONG_FAIL_MSG); + kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.SHORT_SUCCESS_MSG); + kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.SHORT_SUCCESS_MSG); + kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.SHORT_SUCCESS_MSG); + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch3)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch3)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(testTopicListener3.receivedMsgs).containsExactly(expectedReceivedMsgs); + assertThat(testTopicListener3.receivedTopics).containsExactly(expectedReceivedTopics); + + assertThat(myCustomDltProcessor3.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch3.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT3); + } + + @KafkaListener( + id = "4-TopicId", + topics = TEST_TOPIC4, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener4 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + public static final String LONG_SUCCESS_MSG = "5000"; + public static final String SHORT_FAIL_MSG = "1"; + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch4.countDown(); + container.countDownLatch4.countDown(); + return Mono.delay(Duration.ofMillis(Integer.parseInt(message))) // Thread.sleep 대체 + .flatMap(delay -> { + if (message.equals(SHORT_FAIL_MSG)) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + }); + } + } + + + @Test + void longSuccessMsgTwiceThenShortFailMsgTwice(@Autowired TestTopicListener4 topicListener4, + @Autowired MyCustomDltProcessor myCustomDltProcessor4, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("4-TopicId", TEST_TOPIC4); + + String expectedRetryTopic = TEST_TOPIC4 + "-retry"; + String[] expectedReceivedMsgs = { + TestTopicListener4.LONG_SUCCESS_MSG, + TestTopicListener4.LONG_SUCCESS_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + }; + + String[] expectedReceivedTopics = { + TEST_TOPIC4, + TEST_TOPIC4, + TEST_TOPIC4, + TEST_TOPIC4, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + }; + + String[] expectedDltMsgs = { + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + }; + + // When + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.LONG_SUCCESS_MSG); + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.LONG_SUCCESS_MSG); + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.SHORT_FAIL_MSG); + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.SHORT_FAIL_MSG); + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch4)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch4)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(topicListener4.receivedMsgs).containsExactly(expectedReceivedMsgs); + assertThat(topicListener4.receivedTopics).containsExactly(expectedReceivedTopics); + + assertThat(myCustomDltProcessor4.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch4.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT4); + } + + + + @KafkaListener( + id = "5-TopicId", + topics = TEST_TOPIC5, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener5 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + public static final String LONG_SUCCESS_MSG = "5000"; + public static final String SHORT_FAIL_MSG = "1"; + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch5.countDown(); + container.countDownLatch5.countDown(); + return Mono.delay(Duration.ofMillis(Integer.parseInt(message))) // Thread.sleep 대체 + .flatMap(delay -> { + if (message.startsWith(SHORT_FAIL_MSG)) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + }); + } + } + + + @Test + void oneLongSuccessMsgBetweenHunderedShortFailMsg(@Autowired TestTopicListener5 topicListener5, + @Autowired MyCustomDltProcessor myCustomDltProcessor5, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("5-TopicId", TEST_TOPIC5); + + String expectedRetryTopic = TEST_TOPIC5 + "-retry"; + + String[] expectedReceivedMsgs = new String[501]; + for (int i = 0; i < 500; i++) { + expectedReceivedMsgs[i] = TestTopicListener5.SHORT_FAIL_MSG; + } + expectedReceivedMsgs[500] = TestTopicListener5.LONG_SUCCESS_MSG; + + + String[] expectedReceivedTopics = new String[501]; + for (int i = 0; i < 100; i++) { + expectedReceivedTopics[i] = TEST_TOPIC5; + } + for (int i = 100; i < 500; i++) { + expectedReceivedTopics[i] = expectedRetryTopic; + } + expectedReceivedTopics[500] = TEST_TOPIC5; + + + String[] expectedDltMsgs = new String[100]; + for (int i = 0; i < 100; i++) { + expectedDltMsgs[i] = TestTopicListener5.SHORT_FAIL_MSG; + } + + // When + for (int i = 0; i < 100; i++) { + kafkaTemplate.send(TEST_TOPIC5, TestTopicListener5.SHORT_FAIL_MSG); + if (i == 50) { + kafkaTemplate.send(TEST_TOPIC5, TestTopicListener5.LONG_SUCCESS_MSG); + } + } + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch5)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch5)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(topicListener5.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(topicListener5.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); + + assertThat(myCustomDltProcessor5.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch5.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT5); + } + + + @KafkaListener( + id = "6-TopicId", + topics = TEST_TOPIC6, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener6 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); + + public static final String SUCCESS_SUFFIX = ",s"; + public static final String FAIL_SUFFIX = ",f"; + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + + container.extraCountDownLatch6.countDown(); + container.countDownLatch6.countDown(); + + String[] split = message.split(","); + String sleepAWhile = split[0]; + String failOrSuccess = split[1]; + + return Mono.delay(Duration.ofMillis(Integer.parseInt(sleepAWhile))) // Thread.sleep 대체 + .flatMap(delay -> { + if (failOrSuccess.equals("f")) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + }); + } + } + + + @Test + void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime(@Autowired TestTopicListener6 topicListener6, + @Autowired MyCustomDltProcessor myCustomDltProcessor6, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("6-TopicId", TEST_TOPIC6); + + String expectedRetryTopic = TEST_TOPIC6 + "-retry"; + + Random random = new Random(); + ConcurrentLinkedQueue q = new ConcurrentLinkedQueue<>(); + + for (int i = 0; i < 50; i++) { + int randomSleepAWhile = random.nextInt(1, 100); + String msg = String.valueOf(randomSleepAWhile) + TestTopicListener6.SUCCESS_SUFFIX; + q.add(msg); + } + + for (int i = 0; i < 50; i++) { + int randomSleepAWhile = random.nextInt(1, 100); + String msg = String.valueOf(randomSleepAWhile) + TestTopicListener6.FAIL_SUFFIX; + q.add(msg); + } + + int expectedSuccessMsgCount = 50; + int expectedFailedMsgCount = 250; + + int expectedReceivedOriginalTopicCount = 100; + int expectedReceivedRetryTopicCount = 200; + int expectedReceivedDltMsgCount = 50; + + + // When + while (!q.isEmpty()) { + String successOrFailMsg = q.poll(); + kafkaTemplate.send(TEST_TOPIC6, successOrFailMsg); + } + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch6)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch6)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + + long actualReceivedSuccessMsgCount = topicListener6.receivedMsgs.stream() + .map(s -> s.split(",")[1]) + .filter(m -> (',' + m).equals(TestTopicListener6.SUCCESS_SUFFIX)) + .count(); + + long actualReceivedFailedMsgCount = topicListener6.receivedMsgs.stream() + .map(s -> s.split(",")[1]) + .filter(m -> (',' + m).equals( + TestTopicListener6.FAIL_SUFFIX)) + .count(); + + + long actualReceivedOriginalTopicMsgCount = topicListener6.receivedTopics.stream() + .filter(topic -> topic.equals(TEST_TOPIC6)) + .count(); + + long actualReceivedRetryTopicMsgCount = topicListener6.receivedTopics.stream() + .filter(topic -> topic.equals(expectedRetryTopic)) + .count(); + + assertThat(actualReceivedSuccessMsgCount).isEqualTo(expectedSuccessMsgCount); + assertThat(actualReceivedFailedMsgCount).isEqualTo(expectedFailedMsgCount); + assertThat(actualReceivedOriginalTopicMsgCount).isEqualTo(expectedReceivedOriginalTopicCount); + assertThat(actualReceivedRetryTopicMsgCount).isEqualTo(expectedReceivedRetryTopicCount); + + assertThat(myCustomDltProcessor6.receivedMsg.size()).isEqualTo(expectedReceivedDltMsgCount); + assertThat(latchContainer.extraDltCountdownLatch6.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT6); + } + + + + static class CountDownLatchContainer { + + static int COUNT0 = 15; + + static int DLT_COUNT0 = 3; + + CountDownLatch countDownLatch0 = new CountDownLatch(COUNT0); + + CountDownLatch extraCountDownLatch0 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch0 = new CountDownLatch(DLT_COUNT0); + + CountDownLatch extraDltCountdownLatch0 = new CountDownLatch(1000); + + static int COUNT1 = 6; + + static int DLT_COUNT1 = 1; + + CountDownLatch countDownLatch1 = new CountDownLatch(COUNT1); + + CountDownLatch extraCountDownLatch1 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch1 = new CountDownLatch(DLT_COUNT1); + + CountDownLatch extraDltCountdownLatch1 = new CountDownLatch(1000); + + static int COUNT2 = 6; + + static int DLT_COUNT2 = 1; + + CountDownLatch countDownLatch2 = new CountDownLatch(COUNT2); + + CountDownLatch extraCountDownLatch2 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch2 = new CountDownLatch(DLT_COUNT2); + + CountDownLatch extraDltCountdownLatch2 = new CountDownLatch(1000); + + static int COUNT3 = 13; + + static int DLT_COUNT3 = 2; + + CountDownLatch countDownLatch3 = new CountDownLatch(COUNT3); + + CountDownLatch extraCountDownLatch3 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch3 = new CountDownLatch(DLT_COUNT3); + + CountDownLatch extraDltCountdownLatch3 = new CountDownLatch(1000); + + + static int COUNT4 = 12; + + static int DLT_COUNT4 = 2; + + CountDownLatch countDownLatch4 = new CountDownLatch(COUNT4); + + CountDownLatch extraCountDownLatch4 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch4 = new CountDownLatch(DLT_COUNT4); + + CountDownLatch extraDltCountdownLatch4 = new CountDownLatch(1000); + + + static int COUNT5 = 501; + + static int DLT_COUNT5 = 100; + + CountDownLatch countDownLatch5 = new CountDownLatch(COUNT5); + + CountDownLatch extraCountDownLatch5 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch5 = new CountDownLatch(DLT_COUNT5); + + CountDownLatch extraDltCountdownLatch5 = new CountDownLatch(1000); + + + static int COUNT6 = 250; + + static int DLT_COUNT6 = 50; + + CountDownLatch countDownLatch6 = new CountDownLatch(COUNT6); + + CountDownLatch extraCountDownLatch6 = new CountDownLatch(1000); + + CountDownLatch dltCountdownLatch6 = new CountDownLatch(DLT_COUNT6); + + CountDownLatch extraDltCountdownLatch6 = new CountDownLatch(1000); + + CountDownLatch customErrorHandlerCountdownLatch = new CountDownLatch(6); + + CountDownLatch customMessageConverterCountdownLatch = new CountDownLatch(6); + + } + + static class MyCustomDltProcessor { + + final List receivedMsg = new ArrayList<>(); + + public MyCustomDltProcessor(KafkaTemplate kafkaTemplate, + CountDownLatch latch, + CountDownLatch extraLatch) { + this.kafkaTemplate = kafkaTemplate; + this.latch = latch; + this.extraLatch = extraLatch; + } + + private final KafkaTemplate kafkaTemplate; + + private final CountDownLatch latch; + + private final CountDownLatch extraLatch; + + public void processDltMessage(String message) { + this.receivedMsg.add(message); + latch.countDown(); + extraLatch.countDown(); + } + } + + @SuppressWarnings("serial") + static class MyRetryException extends RuntimeException { + MyRetryException(String msg) { + super(msg); + } + } + + @SuppressWarnings("serial") + static class MyDontRetryException extends RuntimeException { + MyDontRetryException(String msg) { + super(msg); + } + } + + @Configuration + static class RetryTopicConfigurations extends RetryTopicConfigurationSupport { + + private static final String DLT_METHOD_NAME = "processDltMessage"; + + static RetryTopicConfiguration createRetryTopicConfiguration(KafkaTemplate template, + String topicName, + String dltBeanName) { + return RetryTopicConfigurationBuilder + .newInstance() + .fixedBackOff(50) + .maxAttempts(5) + .concurrency(1) + .useSingleTopicForSameIntervals() + .includeTopic(topicName) + .doNotRetryOnDltFailure() + .dltHandlerMethod(dltBeanName, DLT_METHOD_NAME) + .create(template); + } + + @Bean + RetryTopicConfiguration testRetryTopic0(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC0, + "myCustomDltProcessor0"); + } + + @Bean + RetryTopicConfiguration testRetryTopic1(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC1, + "myCustomDltProcessor1"); + } + + @Bean + RetryTopicConfiguration testRetryTopic2(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC2, + "myCustomDltProcessor2"); + } + + @Bean + RetryTopicConfiguration testRetryTopic3(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC3, + "myCustomDltProcessor3"); + } + + @Bean + RetryTopicConfiguration testRetryTopic4(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC4, + "myCustomDltProcessor4"); + } + + @Bean + RetryTopicConfiguration testRetryTopic5(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC5, + "myCustomDltProcessor5"); + } + + @Bean + RetryTopicConfiguration testRetryTopic6(KafkaTemplate template) { + return createRetryTopicConfiguration(template, + TEST_TOPIC6, + "myCustomDltProcessor6"); + } + + + @Bean + KafkaListenerErrorHandler myCustomErrorHandler( + CountDownLatchContainer container) { + return (message, exception) -> { + container.customErrorHandlerCountdownLatch.countDown(); + throw exception; + }; + } + + @Bean + SmartMessageConverter myCustomMessageConverter( + CountDownLatchContainer container) { + return new CompositeMessageConverter(Collections.singletonList(new GenericMessageConverter())) { + + @Override + public Object fromMessage(Message message, Class targetClass, Object conversionHint) { + container.customMessageConverterCountdownLatch.countDown(); + return super.fromMessage(message, targetClass, conversionHint); + } + }; + } + + @Bean + CountDownLatchContainer latchContainer() { + return new CountDownLatchContainer(); + } + + @Bean + TestTopicListener0 testTopicListener0() { + return new TestTopicListener0(); + } + + @Bean + TestTopicListener1 testTopicListener1() { + return new TestTopicListener1(); + } + + @Bean + TestTopicListener2 testTopicListener2() { + return new TestTopicListener2(); + } + + @Bean + TestTopicListener3 testTopicListener3() { + return new TestTopicListener3(); + } + + @Bean + TestTopicListener4 testTopicListener4() { + return new TestTopicListener4(); + } + + @Bean + TestTopicListener5 testTopicListener5() { + return new TestTopicListener5(); + } + + @Bean + TestTopicListener6 testTopicListener6() { + return new TestTopicListener6(); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor0(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch0, + latchContainer.extraDltCountdownLatch0); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor1(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch1, + latchContainer.extraDltCountdownLatch1); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor2(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch2, + latchContainer.extraDltCountdownLatch2); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor3(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch3, + latchContainer.extraDltCountdownLatch3); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor4(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch4, + latchContainer.extraDltCountdownLatch4); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor5(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch5, + latchContainer.extraDltCountdownLatch5); + } + + @Bean + MyCustomDltProcessor myCustomDltProcessor6(KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { + return new MyCustomDltProcessor(kafkaTemplate, + latchContainer.dltCountdownLatch6, + latchContainer.extraDltCountdownLatch6); + } + + } + + @Configuration + static class KafkaProducerConfig { + + @Autowired + EmbeddedKafkaBroker broker; + + @Bean + ProducerFactory producerFactory() { + Map configProps = new HashMap<>(); + configProps.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.broker.getBrokersAsString()); + configProps.put( + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + configProps.put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + return new DefaultKafkaProducerFactory<>(configProps); + } + + @Bean("customKafkaTemplate") + KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); + } + } + + @EnableKafka + @Configuration + static class KafkaConsumerConfig { + + @Autowired + EmbeddedKafkaBroker broker; + + @Bean + KafkaAdmin kafkaAdmin() { + Map configs = new HashMap<>(); + configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, this.broker.getBrokersAsString()); + return new KafkaAdmin(configs); + } + + + @Bean + ConsumerFactory consumerFactory() { + Map props = new HashMap<>(); + props.put( + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + this.broker.getBrokersAsString()); + props.put( + ConsumerConfig.GROUP_ID_CONFIG, + "groupId"); + props.put( + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class); + props.put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class); + props.put( + ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + + return new DefaultKafkaConsumerFactory<>(props); + } + + @Bean + ConcurrentKafkaListenerContainerFactory retryTopicListenerContainerFactory( + ConsumerFactory consumerFactory) { + + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + ContainerProperties props = factory.getContainerProperties(); + props.setIdleEventInterval(100L); + props.setPollTimeout(50L); + props.setIdlePartitionEventInterval(100L); + factory.setConsumerFactory(consumerFactory); + factory.setConcurrency(1); + factory.setContainerCustomizer( + container -> container.getContainerProperties().setIdlePartitionEventInterval(100L)); + return factory; + } + + @Bean + ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory( + ConsumerFactory consumerFactory) { + + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(consumerFactory); + factory.setConcurrency(1); + factory.setContainerCustomizer(container -> { + if (container.getListenerId().startsWith("manual")) { + container.getContainerProperties().setAckMode(AckMode.MANUAL); + container.getContainerProperties().setAsyncAcks(true); + } + }); + return factory; + } + + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + + @Bean + ThreadPoolTaskExecutor threadPoolTaskExecutor() { + return new ThreadPoolTaskExecutor(); + } + + } + + private boolean awaitLatch(CountDownLatch latch) { + try { + return latch.await(60, TimeUnit.SECONDS); + } + catch (Exception e) { + fail(e.getMessage()); + throw new RuntimeException(e); + } + } + + private void waitAWhile(ThreadPoolTaskExecutor executor, int sleep) { + CountDownLatch countDownLatch = new CountDownLatch(1); + + Thread thread = new Thread(() -> { + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + countDownLatch.countDown(); + }); + + executor.execute(thread); + assertThat(awaitLatch(countDownLatch)).isTrue(); + } + +} From ac110b4c4fc34b34034dbc6ef91f6e92c0877dd3 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sun, 6 Oct 2024 20:23:04 +0900 Subject: [PATCH 05/30] Add method --- .../listener/adapter/MessagingMessageListenerAdapter.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index edc7c1428e..b433ad1d16 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -690,8 +690,7 @@ protected void asyncFailure(Object request, @Nullable Acknowledgment acknowledgm catch (Throwable ex) { this.logger.error(t, () -> "Future, Mono, or suspend function was completed with an exception for " + source); acknowledge(acknowledgment); - if (request instanceof ConsumerRecord && - ex instanceof RuntimeException) { + if (canAsyncRetry(request, ex)) { ConsumerRecord record = (ConsumerRecord) request; FailedRecordTuple failedRecordTuple = new FailedRecordTuple(record, (RuntimeException) ex); this.callbackForAsyncFailureQueue.accept(failedRecordTuple); @@ -699,6 +698,11 @@ protected void asyncFailure(Object request, @Nullable Acknowledgment acknowledgm } } + private boolean canAsyncRetry(Object request, Throwable exception) { + // The async retry with @RetryableTopic only support in case of SingleRecord Listener. + return request instanceof ConsumerRecord && exception instanceof RuntimeException; + } + protected void handleException(Object records, @Nullable Acknowledgment acknowledgment, Consumer consumer, Message message, ListenerExecutionFailedException e) { From e30319d95199308a9f1a279a8b7dd799a0cc51e5 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Mon, 7 Oct 2024 22:54:19 +0900 Subject: [PATCH 06/30] Fixes lint errors. --- .../KafkaMessageListenerContainer.java | 11 +- .../kafka/listener/MessageListener.java | 3 +- ...fkaBackoffAwareMessageListenerAdapter.java | 3 +- .../MessagingMessageListenerAdapter.java | 13 +- ...RecordMessagingMessageListenerAdapter.java | 2 +- ...eRetryTopicClassLevelIntegrationTests.java | 118 ++- ...pletableFutureRetryTopicScenarioTests.java | 913 +++++++++--------- ...eRetryTopicClassLevelIntegrationTests.java | 151 +-- .../AsyncMonoRetryTopicScenarioTests.java | 788 +++++++-------- 9 files changed, 1056 insertions(+), 946 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index b5c4fd5c2e..a2caae5b56 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -108,7 +108,6 @@ import org.springframework.kafka.listener.ContainerProperties.AckMode; import org.springframework.kafka.listener.ContainerProperties.AssignmentCommitOption; import org.springframework.kafka.listener.ContainerProperties.EOSMode; -import org.springframework.kafka.listener.FailedRecordTracker.FailedRecord; import org.springframework.kafka.listener.adapter.AsyncRepliesAware; import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.KafkaHeaders; @@ -846,7 +845,6 @@ private final class ListenerConsumer implements SchedulingAwareRunnable, Consume private final ConcurrentLinkedDeque> failedRecords = new ConcurrentLinkedDeque(); - @SuppressWarnings(UNCHECKED) ListenerConsumer(GenericMessageListener listener, ListenerType listenerType, ObservationRegistry observationRegistry) { @@ -904,7 +902,7 @@ else if (listener instanceof MessageListener) { this.observationEnabled = this.containerProperties.isObservationEnabled(); if (!AopUtils.isAopProxy(listener)) { - final java.util.function.Consumer callbackForAsyncFailureQueue = + final java.util.function.Consumer> callbackForAsyncFailureQueue = (fRecord) -> this.failedRecords.addLast(fRecord); this.listener.setCallbackForAsyncFailureQueue(callbackForAsyncFailureQueue); } @@ -1310,10 +1308,11 @@ public void run() { try { handleAsyncFailure(); - } catch (Exception e) { + } + catch (Exception e) { // TODO: Need to improve error handling. // TODO: Need to determine how to handle a failed message. - logger.error("Failed to process re-try messages. "); + this.logger.error("Failed to process re-try messages. "); } try { @@ -1466,7 +1465,7 @@ protected void handleAsyncFailure() { if (!copyFailedRecords.isEmpty()) { copyFailedRecords.forEach(failedRecordTuple -> - this.invokeErrorHandlerBySingleRecord(failedRecordTuple.record(), failedRecordTuple.ex())); + this.invokeErrorHandlerBySingleRecord(failedRecordTuple.record(), failedRecordTuple.ex())); } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java index 388478a492..36d0389510 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java @@ -19,6 +19,7 @@ import java.util.function.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; + import org.springframework.kafka.core.FailedRecordTuple; /** @@ -34,7 +35,7 @@ @FunctionalInterface public interface MessageListener extends GenericMessageListener> { - default void setCallbackForAsyncFailureQueue(Consumer asyncRetryCallback) { + default void setCallbackForAsyncFailureQueue(Consumer> asyncRetryCallback) { // } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java index ce38a17cfd..9a28ec3ebe 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java @@ -20,7 +20,6 @@ import java.time.Clock; import java.time.Instant; import java.util.Optional; -import java.util.function.BiConsumer; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -148,7 +147,7 @@ public void onMessage(ConsumerRecord data, Consumer consumer) { } @Override - public void setCallbackForAsyncFailureQueue(java.util.function.Consumer callbackForAsyncFailureQueue) { + public void setCallbackForAsyncFailureQueue(java.util.function.Consumer> callbackForAsyncFailureQueue) { this.delegate.setCallbackForAsyncFailureQueue(callbackForAsyncFailureQueue); } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index b433ad1d16..ec84651755 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -158,7 +158,7 @@ public abstract class MessagingMessageListenerAdapter implements ConsumerS private BiConsumer, RuntimeException> asyncRetryCallback; - private java.util.function.Consumer callbackForAsyncFailureQueue; + private java.util.function.Consumer> callbackForAsyncFailureQueue; /** * Create an instance with the provided bean and method. @@ -167,7 +167,6 @@ public abstract class MessagingMessageListenerAdapter implements ConsumerS */ protected MessagingMessageListenerAdapter(Object bean, Method method) { this(bean, method, null); - System.out.println("here"); } /** @@ -692,7 +691,7 @@ protected void asyncFailure(Object request, @Nullable Acknowledgment acknowledgm acknowledge(acknowledgment); if (canAsyncRetry(request, ex)) { ConsumerRecord record = (ConsumerRecord) request; - FailedRecordTuple failedRecordTuple = new FailedRecordTuple(record, (RuntimeException) ex); + FailedRecordTuple failedRecordTuple = new FailedRecordTuple<>(record, (RuntimeException) ex); this.callbackForAsyncFailureQueue.accept(failedRecordTuple); } } @@ -897,6 +896,10 @@ private boolean rawByParameterIsType(Type parameterType, Type type) { return parameterType instanceof ParameterizedType pType && pType.getRawType().equals(type); } + public void putInAsyncFailureQueue(java.util.function.Consumer> callbackForAsyncFailureQueue) { + this.callbackForAsyncFailureQueue = callbackForAsyncFailureQueue; + } + /** * Root object for reply expression evaluation. * @param request the request. @@ -915,8 +918,4 @@ public void acknowledge() { } - public void putInAsyncFailureQueue(java.util.function.Consumer callbackForAsyncFailureQueue) { - this.callbackForAsyncFailureQueue = callbackForAsyncFailureQueue; - } - } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java index 24f14241c2..e006eb051f 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java @@ -88,7 +88,7 @@ public void onMessage(ConsumerRecord record, @Nullable Acknowledgment ackn @Override public void setCallbackForAsyncFailureQueue( - java.util.function.Consumer asyncRetryCallback) { + java.util.function.Consumer> asyncRetryCallback) { putInAsyncFailureQueue(asyncRetryCallback); } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java index 9564ecf810..d5b2d301b5 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java @@ -47,6 +47,7 @@ import org.apache.kafka.common.serialization.StringSerializer; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.Test; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -94,12 +95,13 @@ @SpringJUnitConfig @DirtiesContext -@EmbeddedKafka(topics = { AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.FIRST_TOPIC, - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.SECOND_TOPIC, - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.THIRD_TOPIC, - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.FOURTH_TOPIC, - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.TWO_LISTENERS_TOPIC, - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.MANUAL_TOPIC }) +@EmbeddedKafka(topics = { + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.FIRST_TOPIC, + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.SECOND_TOPIC, + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.THIRD_TOPIC, + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.FOURTH_TOPIC, + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.TWO_LISTENERS_TOPIC, + AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.MANUAL_TOPIC }) @TestPropertySource(properties = { "five.attempts=5", "kafka.template=customKafkaTemplate"}) public class AsyncCompletableFutureRetryTopicClassLevelIntegrationTests { @@ -167,8 +169,9 @@ void shouldRetrySecondTopic() { } @Test - void shouldRetryThirdTopicWithTimeout(@Autowired KafkaAdmin admin, - @Autowired KafkaListenerEndpointRegistry registry) throws Exception { + void shouldRetryThirdTopicWithTimeout( + @Autowired KafkaAdmin admin, + @Autowired KafkaListenerEndpointRegistry registry) throws Exception { kafkaTemplate.send(THIRD_TOPIC, "Testing topic 3"); assertThat(awaitLatch(latchContainer.countDownLatch3)).isTrue(); @@ -215,26 +218,34 @@ void shouldRetryFourthTopicWithNoDlt() { } @Test - void shouldRetryFifthTopicWithTwoListenersAndManualAssignment(@Autowired - FifthTopicListener1 listener1, - @Autowired FifthTopicListener2 listener2) { + void shouldRetryFifthTopicWithTwoListenersAndManualAssignment( + @Autowired FifthTopicListener1 listener1, + @Autowired FifthTopicListener2 listener2) { kafkaTemplate.send(TWO_LISTENERS_TOPIC, 0, "0", "Testing topic 5 - 0"); kafkaTemplate.send(TWO_LISTENERS_TOPIC, 1, "0", "Testing topic 5 - 1"); assertThat(awaitLatch(latchContainer.countDownLatch51)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatch52)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatchDltThree)).isTrue(); - assertThat(listener1.topics).containsExactly(TWO_LISTENERS_TOPIC, TWO_LISTENERS_TOPIC - + "-listener1-0", TWO_LISTENERS_TOPIC + "-listener1-1", TWO_LISTENERS_TOPIC + "-listener1-2", - TWO_LISTENERS_TOPIC + "-listener1-dlt"); - assertThat(listener2.topics).containsExactly(TWO_LISTENERS_TOPIC, TWO_LISTENERS_TOPIC - + "-listener2-0", TWO_LISTENERS_TOPIC + "-listener2-1", TWO_LISTENERS_TOPIC + "-listener2-2", - TWO_LISTENERS_TOPIC + "-listener2-dlt"); + assertThat(listener1.topics).containsExactly( + TWO_LISTENERS_TOPIC, + TWO_LISTENERS_TOPIC + "-listener1-0", + TWO_LISTENERS_TOPIC + "-listener1-1", + TWO_LISTENERS_TOPIC + "-listener1-2", + TWO_LISTENERS_TOPIC + "-listener1-dlt" + ); + assertThat(listener2.topics).containsExactly( + TWO_LISTENERS_TOPIC, + TWO_LISTENERS_TOPIC + "-listener2-0", + TWO_LISTENERS_TOPIC + "-listener2-1", + TWO_LISTENERS_TOPIC + "-listener2-2", + TWO_LISTENERS_TOPIC + "-listener2-dlt"); } @Test - void shouldRetryManualTopicWithDefaultDlt(@Autowired KafkaListenerEndpointRegistry registry, - @Autowired ConsumerFactory cf) { + void shouldRetryManualTopicWithDefaultDlt( + @Autowired KafkaListenerEndpointRegistry registry, + @Autowired ConsumerFactory cf) { kafkaTemplate.send(MANUAL_TOPIC, "Testing topic 6"); assertThat(awaitLatch(latchContainer.countDownLatch6)).isTrue(); @@ -243,9 +254,10 @@ void shouldRetryManualTopicWithDefaultDlt(@Autowired KafkaListenerEndpointRegist .forEach(id -> { ConcurrentMessageListenerContainer container = (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); - assertThat(container).extracting("commonErrorHandler") - .extracting("seekAfterError", InstanceOfAssertFactories.BOOLEAN) - .isFalse(); + assertThat(container) + .extracting("commonErrorHandler") + .extracting("seekAfterError", InstanceOfAssertFactories.BOOLEAN) + .isFalse(); }); Consumer consumer = cf.createConsumer("manual-dlt", ""); Set tp = @@ -276,14 +288,21 @@ void shouldFirstReuseRetryTopic(@Autowired assertThat(awaitLatch(latchContainer.countDownLatchReuseOne)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatchReuseTwo)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatchReuseThree)).isTrue(); - assertThat(listener1.topics).containsExactly(FIRST_REUSE_RETRY_TOPIC, - FIRST_REUSE_RETRY_TOPIC + "-retry"); - assertThat(listener2.topics).containsExactly(SECOND_REUSE_RETRY_TOPIC, - SECOND_REUSE_RETRY_TOPIC + "-retry-30", SECOND_REUSE_RETRY_TOPIC + "-retry-60", - SECOND_REUSE_RETRY_TOPIC + "-retry-100", SECOND_REUSE_RETRY_TOPIC + "-retry-100"); - assertThat(listener3.topics).containsExactly(THIRD_REUSE_RETRY_TOPIC, - THIRD_REUSE_RETRY_TOPIC + "-retry", THIRD_REUSE_RETRY_TOPIC + "-retry", - THIRD_REUSE_RETRY_TOPIC + "-retry", THIRD_REUSE_RETRY_TOPIC + "-retry"); + assertThat(listener1.topics).containsExactly( + FIRST_REUSE_RETRY_TOPIC, + FIRST_REUSE_RETRY_TOPIC + "-retry"); + assertThat(listener2.topics).containsExactly( + SECOND_REUSE_RETRY_TOPIC, + SECOND_REUSE_RETRY_TOPIC + "-retry-30", + SECOND_REUSE_RETRY_TOPIC + "-retry-60", + SECOND_REUSE_RETRY_TOPIC + "-retry-100", + SECOND_REUSE_RETRY_TOPIC + "-retry-100"); + assertThat(listener3.topics).containsExactly( + THIRD_REUSE_RETRY_TOPIC, + THIRD_REUSE_RETRY_TOPIC + "-retry", + THIRD_REUSE_RETRY_TOPIC + "-retry", + THIRD_REUSE_RETRY_TOPIC + "-retry", + THIRD_REUSE_RETRY_TOPIC + "-retry"); } @Test @@ -324,7 +343,8 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEI container.countDownLatch1.countDown(); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Woooops... in topic " + receivedTopic); @@ -345,7 +365,8 @@ public CompletableFuture listenAgain(String message, @Header(KafkaHeaders. container.countDownIfNotKnown(receivedTopic, container.countDownLatch2); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new IllegalStateException("Another woooops... " + receivedTopic); @@ -377,7 +398,8 @@ public CompletableFuture listenWithAnnotation(String message, @Header(Kafk container.countDownIfNotKnown(receivedTopic, container.countDownLatch3); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new MyRetryException("Annotated woooops... " + receivedTopic); @@ -405,7 +427,8 @@ public CompletableFuture listenNoDlt(String message, @Header(KafkaHeaders. container.countDownIfNotKnown(receivedTopic, container.countDownLatch4); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new IllegalStateException("Another woooops... " + receivedTopic); @@ -456,7 +479,8 @@ public CompletableFuture listenWithAnnotation(String message, @Header(Kafk container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Annotated woooops... " + receivedTopic); @@ -488,7 +512,8 @@ public CompletableFuture listenWithAnnotation2(String message, @Header(Kaf container.countDownLatch52.countDown(); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Annotated woooops... " + receivedTopic); @@ -511,13 +536,16 @@ static class SixthTopicDefaultDLTListener { CountDownLatchContainer container; @KafkaHandler - public CompletableFuture listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic, - @SuppressWarnings("unused") Acknowledgment ack) { + public CompletableFuture listenNoDlt( + String message, + @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic, + @SuppressWarnings("unused") Acknowledgment ack) { return CompletableFuture.supplyAsync(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch6); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new IllegalStateException("Another woooops... " + receivedTopic); @@ -545,7 +573,8 @@ public CompletableFuture listenWithAnnotation2(String message, @Header(K container.countDownIfNotKnown(receivedTopic, container.countDownLatchNoRetry); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); @@ -579,7 +608,8 @@ public CompletableFuture listen1(String message, @Header(KafkaHeaders.RECE container.countDownLatchReuseOne.countDown(); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Another woooops... " + receivedTopic); @@ -609,7 +639,8 @@ public CompletableFuture listen2(String message, @Header(KafkaHeaders.RECE container.countDownLatchReuseTwo.countDown(); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Another woooops... " + receivedTopic); @@ -635,7 +666,8 @@ public CompletableFuture listen3(String message, @Header(KafkaHeaders.RECE container.countDownLatchReuseThree.countDown(); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Another woooops... " + receivedTopic); diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java index 6bc7079d99..98c46c2eb2 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -36,6 +36,7 @@ import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.Test; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -43,7 +44,6 @@ import org.springframework.kafka.annotation.KafkaHandler; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.config.KafkaListenerEndpointRegistry; import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.kafka.core.DefaultKafkaProducerFactory; @@ -95,7 +95,6 @@ public class AsyncCompletableFutureRetryTopicScenarioTests { public final static String TEST_TOPIC6 = "myRetryTopic6"; - @Autowired private KafkaTemplate kafkaTemplate; @@ -105,44 +104,11 @@ public class AsyncCompletableFutureRetryTopicScenarioTests { @Autowired DestinationTopicContainer topicContainer; - @KafkaListener( - id = "0-topicId", - topics = TEST_TOPIC0, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener0 { - - @Autowired - CountDownLatchContainer container; - - private final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - @KafkaHandler - public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch0.countDown(); - container.countDownLatch0.countDown(); - try { - Thread.sleep(1); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - throw new RuntimeException("Woooops... in topic " + receivedTopic); - }); - } - - } - @Test - void allFailCaseTest(@Autowired KafkaListenerEndpointRegistry registry, - @Autowired TestTopicListener0 zeroTopicListener, - @Autowired MyCustomDltProcessor myCustomDltProcessor0, - @Autowired ThreadPoolTaskExecutor executor) { + void allFailCaseTest( + @Autowired TestTopicListener0 zeroTopicListener, + @Autowired MyCustomDltProcessor myCustomDltProcessor0, + @Autowired ThreadPoolTaskExecutor executor) { // All Fail case. String shortFailedMsg1 = "0"; String shortFailedMsg2 = "1"; @@ -211,47 +177,11 @@ void allFailCaseTest(@Autowired KafkaListenerEndpointRegistry registry, assertThat(latchContainer.extraDltCountdownLatch0.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT0); } - @KafkaListener( - id = "1-topicId", - topics = TEST_TOPIC1, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener1 { - - @Autowired - CountDownLatchContainer container; - - private final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - - @KafkaHandler - public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch1.countDown(); - container.countDownLatch1.countDown(); - try { - Thread.sleep(Integer.parseInt(message)); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - if (message.equals("1")) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } - return "Task Completed"; - }); - } - - } - @Test - void firstShortFailAndLastLongSuccessRetryTest(@Autowired TestTopicListener1 testTopicListener1, - @Autowired MyCustomDltProcessor myCustomDltProcessor1, - @Autowired ThreadPoolTaskExecutor executor) { + void firstShortFailAndLastLongSuccessRetryTest( + @Autowired TestTopicListener1 testTopicListener1, + @Autowired MyCustomDltProcessor myCustomDltProcessor1, + @Autowired ThreadPoolTaskExecutor executor) { // Given String longSuccessMsg = "3"; String shortFailedMsg = "1"; @@ -300,48 +230,11 @@ void firstShortFailAndLastLongSuccessRetryTest(@Autowired TestTopicListener1 tes assertThat(latchContainer.extraDltCountdownLatch1.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT1); } - @KafkaListener( - id = "2-topicId", - topics = TEST_TOPIC2, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener2 { - - @Autowired - CountDownLatchContainer container; - - protected final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - @KafkaHandler - public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - - return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch2.countDown(); - container.countDownLatch2.countDown(); - long count = container.countDownLatch2.getCount(); - try { - Thread.sleep(Integer.parseInt(message)); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - if (message.equals("1")) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } - return "Task Completed"; - }); - } - } - @Test - void firstLongSuccessAndLastShortFailed(@Autowired TestTopicListener2 zero2TopicListener, - @Autowired MyCustomDltProcessor myCustomDltProcessor2, - @Autowired ThreadPoolTaskExecutor executor) { + void firstLongSuccessAndLastShortFailed( + @Autowired TestTopicListener2 zero2TopicListener, + @Autowired MyCustomDltProcessor myCustomDltProcessor2, + @Autowired ThreadPoolTaskExecutor executor) { // Given String shortFailedMsg = "1"; String longSuccessMsg = "3"; @@ -389,54 +282,11 @@ void firstLongSuccessAndLastShortFailed(@Autowired TestTopicListener2 zero2Topic assertThat(latchContainer.extraDltCountdownLatch2.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT2); } - - @KafkaListener( - id = "3-topicId", - topics = TEST_TOPIC3, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener3 { - - @Autowired - CountDownLatchContainer container; - - protected final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - public static final String LONG_FAIL_MSG = "5000"; - public static final String SHORT_SUCCESS_MSG = "1"; - - - @KafkaHandler - public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - - return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch3.countDown(); - container.countDownLatch3.countDown(); - long count = container.countDownLatch3.getCount(); - try { - Thread.sleep(Integer.parseInt(message)); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - if (message.equals(LONG_FAIL_MSG)) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } - return "Task Completed"; - }); - } - } - - @Test - void longFailMsgTwiceThenShortSucessMsgThird(@Autowired TestTopicListener3 testTopicListener3, - @Autowired MyCustomDltProcessor myCustomDltProcessor3, - @Autowired ThreadPoolTaskExecutor executor) { + void longFailMsgTwiceThenShortSucessMsgThird( + @Autowired TestTopicListener3 testTopicListener3, + @Autowired MyCustomDltProcessor myCustomDltProcessor3, + @Autowired ThreadPoolTaskExecutor executor) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("3-topicId", TEST_TOPIC3); @@ -485,18 +335,397 @@ void longFailMsgTwiceThenShortSucessMsgThird(@Autowired TestTopicListener3 testT kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.SHORT_SUCCESS_MSG); kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.SHORT_SUCCESS_MSG); - // Then - assertThat(awaitLatch(latchContainer.countDownLatch3)).isTrue(); - assertThat(awaitLatch(latchContainer.dltCountdownLatch3)).isTrue(); + // Then + assertThat(awaitLatch(latchContainer.countDownLatch3)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch3)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(testTopicListener3.receivedMsgs).containsExactly(expectedReceivedMsgs); + assertThat(testTopicListener3.receivedTopics).containsExactly(expectedReceivedTopics); + + assertThat(myCustomDltProcessor3.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch3.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT3); + } + + @Test + void longSuccessMsgTwiceThenShortFailMsgTwice( + @Autowired TestTopicListener4 topicListener4, + @Autowired MyCustomDltProcessor myCustomDltProcessor4, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("4-TopicId", TEST_TOPIC4); + + String expectedRetryTopic = TEST_TOPIC4 + "-retry"; + String[] expectedReceivedMsgs = { + TestTopicListener4.LONG_SUCCESS_MSG, + TestTopicListener4.LONG_SUCCESS_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + }; + + String[] expectedReceivedTopics = { + TEST_TOPIC4, + TEST_TOPIC4, + TEST_TOPIC4, + TEST_TOPIC4, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + expectedRetryTopic, + }; + + String[] expectedDltMsgs = { + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + }; + + // When + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.LONG_SUCCESS_MSG); + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.LONG_SUCCESS_MSG); + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.SHORT_FAIL_MSG); + kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.SHORT_FAIL_MSG); + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch4)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch4)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(topicListener4.receivedMsgs).containsExactly(expectedReceivedMsgs); + assertThat(topicListener4.receivedTopics).containsExactly(expectedReceivedTopics); + + assertThat(myCustomDltProcessor4.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch4.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT4); + } + + @Test + void oneLongSuccessMsgBetweenHunderedShortFailMsg( + @Autowired TestTopicListener5 topicListener5, + @Autowired MyCustomDltProcessor myCustomDltProcessor5, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("5-TopicId", TEST_TOPIC5); + + String expectedRetryTopic = TEST_TOPIC5 + "-retry"; + + String[] expectedReceivedMsgs = new String[501]; + for (int i = 0; i < 500; i++) { + expectedReceivedMsgs[i] = TestTopicListener5.SHORT_FAIL_MSG; + } + expectedReceivedMsgs[500] = TestTopicListener5.LONG_SUCCESS_MSG; + + + String[] expectedReceivedTopics = new String[501]; + for (int i = 0; i < 100; i++) { + expectedReceivedTopics[i] = TEST_TOPIC5; + } + for (int i = 100; i < 500; i++) { + expectedReceivedTopics[i] = expectedRetryTopic; + } + expectedReceivedTopics[500] = TEST_TOPIC5; + + + String[] expectedDltMsgs = new String[100]; + for (int i = 0; i < 100; i++) { + expectedDltMsgs[i] = TestTopicListener5.SHORT_FAIL_MSG; + } + + // When + for (int i = 0; i < 100; i++) { + kafkaTemplate.send(TEST_TOPIC5, TestTopicListener5.SHORT_FAIL_MSG); + if (i == 50) { + kafkaTemplate.send(TEST_TOPIC5, TestTopicListener5.LONG_SUCCESS_MSG); + } + } + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch5)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch5)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + assertThat(topicListener5.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(topicListener5.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); + + assertThat(myCustomDltProcessor5.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); + assertThat(latchContainer.extraDltCountdownLatch5.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT5); + } + + @Test + void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime( + @Autowired TestTopicListener6 topicListener6, + @Autowired MyCustomDltProcessor myCustomDltProcessor6, + @Autowired ThreadPoolTaskExecutor executor) { + // Given + DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("6-TopicId", TEST_TOPIC6); + + String expectedRetryTopic = TEST_TOPIC6 + "-retry"; + + Random random = new Random(); + ConcurrentLinkedQueue q = new ConcurrentLinkedQueue<>(); + + for (int i = 0; i < 50; i++) { + int randomSleepAWhile = random.nextInt(1, 100); + String msg = String.valueOf(randomSleepAWhile) + TestTopicListener6.SUCCESS_SUFFIX; + q.add(msg); + } + + for (int i = 0; i < 50; i++) { + int randomSleepAWhile = random.nextInt(1, 100); + String msg = String.valueOf(randomSleepAWhile) + TestTopicListener6.FAIL_SUFFIX; + q.add(msg); + } + + int expectedSuccessMsgCount = 50; + int expectedFailedMsgCount = 250; + + int expectedReceivedOriginalTopicCount = 100; + int expectedReceivedRetryTopicCount = 200; + int expectedReceivedDltMsgCount = 50; + + + // When + while (!q.isEmpty()) { + String successOrFailMsg = q.poll(); + kafkaTemplate.send(TEST_TOPIC6, successOrFailMsg); + } + + // Then + assertThat(awaitLatch(latchContainer.countDownLatch6)).isTrue(); + assertThat(awaitLatch(latchContainer.dltCountdownLatch6)).isTrue(); + + waitAWhile(executor, 10000); + + assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); + + long actualReceivedSuccessMsgCount = topicListener6.receivedMsgs.stream() + .map(s -> s.split(",")[1]) + .filter(m -> (',' + m).equals(TestTopicListener6.SUCCESS_SUFFIX)) + .count(); + + long actualReceivedFailedMsgCount = topicListener6.receivedMsgs.stream() + .map(s -> s.split(",")[1]) + .filter(m -> (',' + m).equals( + TestTopicListener6.FAIL_SUFFIX)) + .count(); + + + long actualReceivedOriginalTopicMsgCount = topicListener6.receivedTopics.stream() + .filter(topic -> topic.equals(TEST_TOPIC6)) + .count(); + + long actualReceivedRetryTopicMsgCount = topicListener6.receivedTopics.stream() + .filter(topic -> topic.equals(expectedRetryTopic)) + .count(); + + assertThat(actualReceivedSuccessMsgCount).isEqualTo(expectedSuccessMsgCount); + assertThat(actualReceivedFailedMsgCount).isEqualTo(expectedFailedMsgCount); + assertThat(actualReceivedOriginalTopicMsgCount).isEqualTo(expectedReceivedOriginalTopicCount); + assertThat(actualReceivedRetryTopicMsgCount).isEqualTo(expectedReceivedRetryTopicCount); + + assertThat(myCustomDltProcessor6.receivedMsg.size()).isEqualTo(expectedReceivedDltMsgCount); + assertThat(latchContainer.extraDltCountdownLatch6.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT6); + } + + private boolean awaitLatch(CountDownLatch latch) { + try { + return latch.await(60, TimeUnit.SECONDS); + } + catch (Exception e) { + fail(e.getMessage()); + throw new RuntimeException(e); + } + + } + + private void waitAWhile(ThreadPoolTaskExecutor executor, int sleep) { + CountDownLatch countDownLatch = new CountDownLatch(1); + + Thread thread = new Thread(() -> { + try { + Thread.sleep(sleep); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + countDownLatch.countDown(); + }); + + executor.execute(thread); + assertThat(awaitLatch(countDownLatch)).isTrue(); + } + + @KafkaListener( + id = "0-topicId", + topics = TEST_TOPIC0, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener0 { + + @Autowired + CountDownLatchContainer container; + + private final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch0.countDown(); + container.countDownLatch0.countDown(); + try { + Thread.sleep(1); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Woooops... in topic " + receivedTopic); + }); + } + + } + + @KafkaListener( + id = "1-topicId", + topics = TEST_TOPIC1, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener1 { + + @Autowired + CountDownLatchContainer container; + + private final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch1.countDown(); + container.countDownLatch1.countDown(); + try { + Thread.sleep(Integer.parseInt(message)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (message.equals("1")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } + + } + + @KafkaListener( + id = "2-topicId", + topics = TEST_TOPIC2, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener2 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch2.countDown(); + container.countDownLatch2.countDown(); + long count = container.countDownLatch2.getCount(); + try { + Thread.sleep(Integer.parseInt(message)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + + if (message.equals("1")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } + } + + @KafkaListener( + id = "3-topicId", + topics = TEST_TOPIC3, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener3 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + public static final String LONG_FAIL_MSG = "5000"; + + public static final String SHORT_SUCCESS_MSG = "1"; - waitAWhile(executor, 10000); + @KafkaHandler + public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(testTopicListener3.receivedMsgs).containsExactly(expectedReceivedMsgs); - assertThat(testTopicListener3.receivedTopics).containsExactly(expectedReceivedTopics); + return CompletableFuture.supplyAsync(() -> { + container.extraCountDownLatch3.countDown(); + container.countDownLatch3.countDown(); + long count = container.countDownLatch3.getCount(); + try { + Thread.sleep(Integer.parseInt(message)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } - assertThat(myCustomDltProcessor3.receivedMsg).containsExactly(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch3.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT3); + if (message.equals(LONG_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } } @KafkaListener( @@ -512,9 +741,11 @@ static class TestTopicListener4 { CountDownLatchContainer container; protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); public static final String LONG_SUCCESS_MSG = "5000"; + public static final String SHORT_FAIL_MSG = "1"; @KafkaHandler @@ -527,7 +758,8 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC container.countDownLatch4.countDown(); try { Thread.sleep(Integer.parseInt(message)); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } @@ -539,72 +771,6 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC } } - - @Test - void longSuccessMsgTwiceThenShortFailMsgTwice(@Autowired TestTopicListener4 topicListener4, - @Autowired MyCustomDltProcessor myCustomDltProcessor4, - @Autowired ThreadPoolTaskExecutor executor) { - // Given - DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("4-TopicId", TEST_TOPIC4); - - String expectedRetryTopic = TEST_TOPIC4 + "-retry"; - String[] expectedReceivedMsgs = { - TestTopicListener4.LONG_SUCCESS_MSG, - TestTopicListener4.LONG_SUCCESS_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - }; - - String[] expectedReceivedTopics = { - TEST_TOPIC4, - TEST_TOPIC4, - TEST_TOPIC4, - TEST_TOPIC4, - expectedRetryTopic, - expectedRetryTopic, - expectedRetryTopic, - expectedRetryTopic, - expectedRetryTopic, - expectedRetryTopic, - expectedRetryTopic, - expectedRetryTopic, - }; - - String[] expectedDltMsgs = { - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - }; - - // When - kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.LONG_SUCCESS_MSG); - kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.LONG_SUCCESS_MSG); - kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.SHORT_FAIL_MSG); - kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.SHORT_FAIL_MSG); - - // Then - assertThat(awaitLatch(latchContainer.countDownLatch4)).isTrue(); - assertThat(awaitLatch(latchContainer.dltCountdownLatch4)).isTrue(); - - waitAWhile(executor, 10000); - - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(topicListener4.receivedMsgs).containsExactly(expectedReceivedMsgs); - assertThat(topicListener4.receivedTopics).containsExactly(expectedReceivedTopics); - - assertThat(myCustomDltProcessor4.receivedMsg).containsExactly(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch4.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT4); - } - - - @KafkaListener( id = "5-TopicId", topics = TEST_TOPIC5, @@ -618,9 +784,11 @@ static class TestTopicListener5 { CountDownLatchContainer container; protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); public static final String LONG_SUCCESS_MSG = "5000"; + public static final String SHORT_FAIL_MSG = "1"; @KafkaHandler @@ -633,7 +801,8 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC container.countDownLatch5.countDown(); try { Thread.sleep(Integer.parseInt(message)); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } @@ -645,61 +814,6 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC } } - - @Test - void oneLongSuccessMsgBetweenHunderedShortFailMsg(@Autowired TestTopicListener5 topicListener5, - @Autowired MyCustomDltProcessor myCustomDltProcessor5, - @Autowired ThreadPoolTaskExecutor executor) { - // Given - DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("5-TopicId", TEST_TOPIC5); - - String expectedRetryTopic = TEST_TOPIC5 + "-retry"; - - String[] expectedReceivedMsgs = new String[501]; - for (int i = 0; i < 500; i++) { - expectedReceivedMsgs[i] = TestTopicListener5.SHORT_FAIL_MSG; - } - expectedReceivedMsgs[500] = TestTopicListener5.LONG_SUCCESS_MSG; - - - String[] expectedReceivedTopics = new String[501]; - for (int i = 0; i < 100; i++) { - expectedReceivedTopics[i] = TEST_TOPIC5; - } - for (int i = 100; i < 500; i++) { - expectedReceivedTopics[i] = expectedRetryTopic; - } - expectedReceivedTopics[500] = TEST_TOPIC5; - - - String[] expectedDltMsgs = new String[100]; - for (int i = 0; i < 100; i++) { - expectedDltMsgs[i] = TestTopicListener5.SHORT_FAIL_MSG; - } - - // When - for (int i = 0; i < 100; i++) { - kafkaTemplate.send(TEST_TOPIC5, TestTopicListener5.SHORT_FAIL_MSG); - if (i == 50) { - kafkaTemplate.send(TEST_TOPIC5, TestTopicListener5.LONG_SUCCESS_MSG); - } - } - - // Then - assertThat(awaitLatch(latchContainer.countDownLatch5)).isTrue(); - assertThat(awaitLatch(latchContainer.dltCountdownLatch5)).isTrue(); - - waitAWhile(executor, 10000); - - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(topicListener5.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); - assertThat(topicListener5.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - - assertThat(myCustomDltProcessor5.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch5.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT5); - } - - @KafkaListener( id = "6-TopicId", topics = TEST_TOPIC6, @@ -713,9 +827,11 @@ static class TestTopicListener6 { CountDownLatchContainer container; protected final List receivedMsgs = new ArrayList<>(); + private final List receivedTopics = new ArrayList<>(); public static final String SUCCESS_SUFFIX = ",s"; + public static final String FAIL_SUFFIX = ",f"; @KafkaHandler @@ -733,7 +849,8 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC try { Thread.sleep(Integer.parseInt(sleepAWhile)); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } @@ -745,84 +862,6 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC } } - - @Test - void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime(@Autowired TestTopicListener6 topicListener6, - @Autowired MyCustomDltProcessor myCustomDltProcessor6, - @Autowired ThreadPoolTaskExecutor executor) { - // Given - DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("6-TopicId", TEST_TOPIC6); - - String expectedRetryTopic = TEST_TOPIC6 + "-retry"; - - Random random = new Random(); - ConcurrentLinkedQueue q = new ConcurrentLinkedQueue<>(); - - for (int i = 0; i < 50; i++) { - int randomSleepAWhile = random.nextInt(1, 100); - String msg = String.valueOf(randomSleepAWhile) + TestTopicListener6.SUCCESS_SUFFIX; - q.add(msg); - } - - for (int i = 0; i < 50; i++) { - int randomSleepAWhile = random.nextInt(1, 100); - String msg = String.valueOf(randomSleepAWhile) + TestTopicListener6.FAIL_SUFFIX; - q.add(msg); - } - - int expectedSuccessMsgCount = 50; - int expectedFailedMsgCount = 250; - - int expectedReceivedOriginalTopicCount = 100; - int expectedReceivedRetryTopicCount = 200; - int expectedReceivedDltMsgCount = 50; - - - // When - while (!q.isEmpty()) { - String successOrFailMsg = q.poll(); - kafkaTemplate.send(TEST_TOPIC6, successOrFailMsg); - } - - // Then - assertThat(awaitLatch(latchContainer.countDownLatch6)).isTrue(); - assertThat(awaitLatch(latchContainer.dltCountdownLatch6)).isTrue(); - - waitAWhile(executor, 10000); - - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - - long actualReceivedSuccessMsgCount = topicListener6.receivedMsgs.stream() - .map(s -> s.split(",")[1]) - .filter(m -> (',' + m).equals(TestTopicListener6.SUCCESS_SUFFIX)) - .count(); - - long actualReceivedFailedMsgCount = topicListener6.receivedMsgs.stream() - .map(s -> s.split(",")[1]) - .filter(m -> (',' + m).equals( - TestTopicListener6.FAIL_SUFFIX)) - .count(); - - - long actualReceivedOriginalTopicMsgCount = topicListener6.receivedTopics.stream() - .filter(topic -> topic.equals(TEST_TOPIC6)) - .count(); - - long actualReceivedRetryTopicMsgCount = topicListener6.receivedTopics.stream() - .filter(topic -> topic.equals(expectedRetryTopic)) - .count(); - - assertThat(actualReceivedSuccessMsgCount).isEqualTo(expectedSuccessMsgCount); - assertThat(actualReceivedFailedMsgCount).isEqualTo(expectedFailedMsgCount); - assertThat(actualReceivedOriginalTopicMsgCount).isEqualTo(expectedReceivedOriginalTopicCount); - assertThat(actualReceivedRetryTopicMsgCount).isEqualTo(expectedReceivedRetryTopicCount); - - assertThat(myCustomDltProcessor6.receivedMsg.size()).isEqualTo(expectedReceivedDltMsgCount); - assertThat(latchContainer.extraDltCountdownLatch6.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT6); - } - - - static class CountDownLatchContainer { static int COUNT0 = 15; @@ -873,7 +912,6 @@ static class CountDownLatchContainer { CountDownLatch extraDltCountdownLatch3 = new CountDownLatch(1000); - static int COUNT4 = 12; static int DLT_COUNT4 = 2; @@ -886,7 +924,6 @@ static class CountDownLatchContainer { CountDownLatch extraDltCountdownLatch4 = new CountDownLatch(1000); - static int COUNT5 = 501; static int DLT_COUNT5 = 100; @@ -899,7 +936,6 @@ static class CountDownLatchContainer { CountDownLatch extraDltCountdownLatch5 = new CountDownLatch(1000); - static int COUNT6 = 250; static int DLT_COUNT6 = 50; @@ -922,7 +958,7 @@ static class MyCustomDltProcessor { final List receivedMsg = new ArrayList<>(); - public MyCustomDltProcessor(KafkaTemplate kafkaTemplate, + MyCustomDltProcessor(KafkaTemplate kafkaTemplate, CountDownLatch latch, CountDownLatch extraLatch) { this.kafkaTemplate = kafkaTemplate; @@ -979,54 +1015,60 @@ static RetryTopicConfiguration createRetryTopicConfiguration(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC0, - "myCustomDltProcessor0"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC0, + "myCustomDltProcessor0"); } @Bean RetryTopicConfiguration testRetryTopic1(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC1, - "myCustomDltProcessor1"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC1, + "myCustomDltProcessor1"); } @Bean RetryTopicConfiguration testRetryTopic2(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC2, - "myCustomDltProcessor2"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC2, + "myCustomDltProcessor2"); } @Bean RetryTopicConfiguration testRetryTopic3(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC3, - "myCustomDltProcessor3"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC3, + "myCustomDltProcessor3"); } @Bean RetryTopicConfiguration testRetryTopic4(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC4, - "myCustomDltProcessor4"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC4, + "myCustomDltProcessor4"); } @Bean RetryTopicConfiguration testRetryTopic5(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC5, - "myCustomDltProcessor5"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC5, + "myCustomDltProcessor5"); } @Bean RetryTopicConfiguration testRetryTopic6(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC6, - "myCustomDltProcessor6"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC6, + "myCustomDltProcessor6"); } - @Bean KafkaListenerErrorHandler myCustomErrorHandler( CountDownLatchContainer container) { @@ -1090,56 +1132,63 @@ TestTopicListener6 testTopicListener6() { } @Bean - MyCustomDltProcessor myCustomDltProcessor0(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor0( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch0, latchContainer.extraDltCountdownLatch0); } @Bean - MyCustomDltProcessor myCustomDltProcessor1(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor1( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch1, latchContainer.extraDltCountdownLatch1); } @Bean - MyCustomDltProcessor myCustomDltProcessor2(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor2( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch2, latchContainer.extraDltCountdownLatch2); } @Bean - MyCustomDltProcessor myCustomDltProcessor3(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor3( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch3, latchContainer.extraDltCountdownLatch3); } @Bean - MyCustomDltProcessor myCustomDltProcessor4(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor4( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch4, latchContainer.extraDltCountdownLatch4); } @Bean - MyCustomDltProcessor myCustomDltProcessor5(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor5( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch5, latchContainer.extraDltCountdownLatch5); } @Bean - MyCustomDltProcessor myCustomDltProcessor6(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor6( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch6, latchContainer.extraDltCountdownLatch6); @@ -1172,6 +1221,7 @@ ProducerFactory producerFactory() { KafkaTemplate kafkaTemplate() { return new KafkaTemplate<>(producerFactory()); } + } @EnableKafka @@ -1188,7 +1238,6 @@ KafkaAdmin kafkaAdmin() { return new KafkaAdmin(configs); } - @Bean ConsumerFactory consumerFactory() { Map props = new HashMap<>(); @@ -1255,30 +1304,4 @@ ThreadPoolTaskExecutor threadPoolTaskExecutor() { } - private boolean awaitLatch(CountDownLatch latch) { - try { - return latch.await(60, TimeUnit.SECONDS); - } - catch (Exception e) { - fail(e.getMessage()); - throw new RuntimeException(e); - } - } - - private void waitAWhile(ThreadPoolTaskExecutor executor, int sleep) { - CountDownLatch countDownLatch = new CountDownLatch(1); - - Thread thread = new Thread(() -> { - try { - Thread.sleep(sleep); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - countDownLatch.countDown(); - }); - - executor.execute(thread); - assertThat(awaitLatch(countDownLatch)).isTrue(); - } - } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java index 6f27b1fea7..31bb05b930 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 the original author or authors. + * Copyright 2018-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,6 +46,7 @@ import org.apache.kafka.common.serialization.StringSerializer; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.Test; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -94,12 +95,13 @@ @SpringJUnitConfig @DirtiesContext -@EmbeddedKafka(topics = { AsyncMonoFutureRetryTopicClassLevelIntegrationTests.FIRST_TOPIC, - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.SECOND_TOPIC, - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.THIRD_TOPIC, - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.FOURTH_TOPIC, - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.TWO_LISTENERS_TOPIC, - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.MANUAL_TOPIC }) +@EmbeddedKafka(topics = { + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.FIRST_TOPIC, + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.SECOND_TOPIC, + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.THIRD_TOPIC, + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.FOURTH_TOPIC, + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.TWO_LISTENERS_TOPIC, + AsyncMonoFutureRetryTopicClassLevelIntegrationTests.MANUAL_TOPIC }) @TestPropertySource(properties = { "five.attempts=5", "kafka.template=customKafkaTemplate"}) public class AsyncMonoFutureRetryTopicClassLevelIntegrationTests { @@ -167,8 +169,9 @@ void shouldRetrySecondTopic() { } @Test - void shouldRetryThirdTopicWithTimeout(@Autowired KafkaAdmin admin, - @Autowired KafkaListenerEndpointRegistry registry) throws Exception { + void shouldRetryThirdTopicWithTimeout( + @Autowired KafkaAdmin admin, + @Autowired KafkaListenerEndpointRegistry registry) throws Exception { kafkaTemplate.send(THIRD_TOPIC, "Testing topic 3"); assertThat(awaitLatch(latchContainer.countDownLatch3)).isTrue(); @@ -215,27 +218,33 @@ void shouldRetryFourthTopicWithNoDlt() { } @Test - void shouldRetryFifthTopicWithTwoListenersAndManualAssignment(@Autowired - FifthTopicListener1 listener1, - @Autowired - FifthTopicListener2 listener2) { + void shouldRetryFifthTopicWithTwoListenersAndManualAssignment( + @Autowired FifthTopicListener1 listener1, + @Autowired FifthTopicListener2 listener2) { kafkaTemplate.send(TWO_LISTENERS_TOPIC, 0, "0", "Testing topic 5 - 0"); kafkaTemplate.send(TWO_LISTENERS_TOPIC, 1, "0", "Testing topic 5 - 1"); assertThat(awaitLatch(latchContainer.countDownLatch51)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatch52)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatchDltThree)).isTrue(); - assertThat(listener1.topics).containsExactly(TWO_LISTENERS_TOPIC, TWO_LISTENERS_TOPIC - + "-listener1-0", TWO_LISTENERS_TOPIC + "-listener1-1", TWO_LISTENERS_TOPIC + "-listener1-2", - TWO_LISTENERS_TOPIC + "-listener1-dlt"); - assertThat(listener2.topics).containsExactly(TWO_LISTENERS_TOPIC, TWO_LISTENERS_TOPIC - + "-listener2-0", TWO_LISTENERS_TOPIC + "-listener2-1", TWO_LISTENERS_TOPIC + "-listener2-2", - TWO_LISTENERS_TOPIC + "-listener2-dlt"); + assertThat(listener1.topics).containsExactly( + TWO_LISTENERS_TOPIC, + TWO_LISTENERS_TOPIC + "-listener1-0", + TWO_LISTENERS_TOPIC + "-listener1-1", + TWO_LISTENERS_TOPIC + "-listener1-2", + TWO_LISTENERS_TOPIC + "-listener1-dlt"); + assertThat(listener2.topics).containsExactly( + TWO_LISTENERS_TOPIC, + TWO_LISTENERS_TOPIC + "-listener2-0", + TWO_LISTENERS_TOPIC + "-listener2-1", + TWO_LISTENERS_TOPIC + "-listener2-2", + TWO_LISTENERS_TOPIC + "-listener2-dlt"); } @Test - void shouldRetryManualTopicWithDefaultDlt(@Autowired KafkaListenerEndpointRegistry registry, - @Autowired ConsumerFactory cf) { + void shouldRetryManualTopicWithDefaultDlt( + @Autowired KafkaListenerEndpointRegistry registry, + @Autowired ConsumerFactory cf) { kafkaTemplate.send(MANUAL_TOPIC, "Testing topic 6"); assertThat(awaitLatch(latchContainer.countDownLatch6)).isTrue(); @@ -244,9 +253,10 @@ void shouldRetryManualTopicWithDefaultDlt(@Autowired KafkaListenerEndpointRegist .forEach(id -> { ConcurrentMessageListenerContainer container = (ConcurrentMessageListenerContainer) registry.getListenerContainer(id); - assertThat(container).extracting("commonErrorHandler") - .extracting("seekAfterError", InstanceOfAssertFactories.BOOLEAN) - .isFalse(); + assertThat(container) + .extracting("commonErrorHandler") + .extracting("seekAfterError", InstanceOfAssertFactories.BOOLEAN) + .isFalse(); }); Consumer consumer = cf.createConsumer("manual-dlt", ""); Set tp = @@ -265,11 +275,10 @@ void shouldRetryManualTopicWithDefaultDlt(@Autowired KafkaListenerEndpointRegist } @Test - void shouldFirstReuseRetryTopic(@Autowired - FirstReuseRetryTopicListener listener1, - @Autowired - SecondReuseRetryTopicListener listener2, @Autowired - ThirdReuseRetryTopicListener listener3) { + void shouldFirstReuseRetryTopic( + @Autowired FirstReuseRetryTopicListener listener1, + @Autowired SecondReuseRetryTopicListener listener2, + @Autowired ThirdReuseRetryTopicListener listener3) { kafkaTemplate.send(FIRST_REUSE_RETRY_TOPIC, "Testing reuse topic 1"); kafkaTemplate.send(SECOND_REUSE_RETRY_TOPIC, "Testing reuse topic 2"); @@ -277,14 +286,21 @@ void shouldFirstReuseRetryTopic(@Autowired assertThat(awaitLatch(latchContainer.countDownLatchReuseOne)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatchReuseTwo)).isTrue(); assertThat(awaitLatch(latchContainer.countDownLatchReuseThree)).isTrue(); - assertThat(listener1.topics).containsExactly(FIRST_REUSE_RETRY_TOPIC, - FIRST_REUSE_RETRY_TOPIC + "-retry"); - assertThat(listener2.topics).containsExactly(SECOND_REUSE_RETRY_TOPIC, - SECOND_REUSE_RETRY_TOPIC + "-retry-30", SECOND_REUSE_RETRY_TOPIC + "-retry-60", - SECOND_REUSE_RETRY_TOPIC + "-retry-100", SECOND_REUSE_RETRY_TOPIC + "-retry-100"); - assertThat(listener3.topics).containsExactly(THIRD_REUSE_RETRY_TOPIC, - THIRD_REUSE_RETRY_TOPIC + "-retry", THIRD_REUSE_RETRY_TOPIC + "-retry", - THIRD_REUSE_RETRY_TOPIC + "-retry", THIRD_REUSE_RETRY_TOPIC + "-retry"); + assertThat(listener1.topics).containsExactly( + FIRST_REUSE_RETRY_TOPIC, + FIRST_REUSE_RETRY_TOPIC + "-retry"); + assertThat(listener2.topics).containsExactly( + SECOND_REUSE_RETRY_TOPIC, + SECOND_REUSE_RETRY_TOPIC + "-retry-30", + SECOND_REUSE_RETRY_TOPIC + "-retry-60", + SECOND_REUSE_RETRY_TOPIC + "-retry-100", + SECOND_REUSE_RETRY_TOPIC + "-retry-100"); + assertThat(listener3.topics).containsExactly( + THIRD_REUSE_RETRY_TOPIC, + THIRD_REUSE_RETRY_TOPIC + "-retry", + THIRD_REUSE_RETRY_TOPIC + "-retry", + THIRD_REUSE_RETRY_TOPIC + "-retry", + THIRD_REUSE_RETRY_TOPIC + "-retry"); } @Test @@ -320,12 +336,15 @@ static class FirstTopicListener { CountDownLatchContainer container; @KafkaHandler - public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + public Mono listen( + String message, + @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { container.countDownLatch1.countDown(); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Woooops... in topic " + receivedTopic); @@ -341,12 +360,15 @@ static class SecondTopicListener { CountDownLatchContainer container; @KafkaHandler - public Mono listenAgain(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + public Mono listenAgain( + String message, + @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch2); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new IllegalStateException("Another woooops... " + receivedTopic); @@ -373,12 +395,15 @@ static class ThirdTopicListener { CountDownLatchContainer container; @KafkaHandler - public Mono listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + public Mono listenWithAnnotation( + String message, + @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch3); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new MyRetryException("Annotated woooops... " + receivedTopic); @@ -401,12 +426,15 @@ static class FourthTopicListener { CountDownLatchContainer container; @KafkaHandler - public Mono listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + public Mono listenNoDlt( + String message, + @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch4); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new IllegalStateException("Another woooops... " + receivedTopic); @@ -448,8 +476,7 @@ public void annotatedDltMethod(ConsumerRecord record) { topicPartitions = {@org.springframework.kafka.annotation.TopicPartition(topic = TWO_LISTENERS_TOPIC, partitionOffsets = @PartitionOffset(partition = "0", initialOffset = "0"))}, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) - static class FifthTopicListener1 extends - AbstractFifthTopicListener { + static class FifthTopicListener1 extends AbstractFifthTopicListener { @KafkaHandler public Mono listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { @@ -458,7 +485,8 @@ public Mono listenWithAnnotation(String message, @Header(KafkaHeaders.RECE container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Annotated woooops... " + receivedTopic); @@ -481,8 +509,7 @@ public Mono listenWithAnnotation(String message, @Header(KafkaHeaders.RECE topicPartitions = {@org.springframework.kafka.annotation.TopicPartition(topic = TWO_LISTENERS_TOPIC, partitionOffsets = @PartitionOffset(partition = "1", initialOffset = "0"))}, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY) - static class FifthTopicListener2 extends - AbstractFifthTopicListener { + static class FifthTopicListener2 extends AbstractFifthTopicListener { @KafkaHandler public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { @@ -491,7 +518,8 @@ public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.REC container.countDownLatch52.countDown(); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Annotated woooops... " + receivedTopic); @@ -514,13 +542,16 @@ static class SixthTopicDefaultDLTListener { CountDownLatchContainer container; @KafkaHandler - public Mono listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic, - @SuppressWarnings("unused") Acknowledgment ack) { + public Mono listenNoDlt( + String message, + @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic, + @SuppressWarnings("unused") Acknowledgment ack) { return Mono.fromCallable(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch6); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new IllegalStateException("Another woooops... " + receivedTopic); @@ -548,7 +579,8 @@ public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.REC container.countDownIfNotKnown(receivedTopic, container.countDownLatchNoRetry); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); @@ -582,7 +614,8 @@ public Mono listen1(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) S container.countDownLatchReuseOne.countDown(); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Another woooops... " + receivedTopic); @@ -612,7 +645,8 @@ public Mono listen2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) S container.countDownLatchReuseTwo.countDown(); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Another woooops... " + receivedTopic); @@ -638,7 +672,8 @@ public Mono listen3(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) S container.countDownLatchReuseThree.countDown(); try { Thread.sleep(1); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException(e); } throw new RuntimeException("Another woooops... " + receivedTopic); diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java index 62623e50ee..49f67beab7 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.Random; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -37,6 +36,7 @@ import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.Test; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -44,7 +44,6 @@ import org.springframework.kafka.annotation.KafkaHandler; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.config.KafkaListenerEndpointRegistry; import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.kafka.core.DefaultKafkaProducerFactory; @@ -98,7 +97,6 @@ public class AsyncMonoRetryTopicScenarioTests { public final static String TEST_TOPIC6 = "myRetryTopic6"; - @Autowired private KafkaTemplate kafkaTemplate; @@ -108,43 +106,11 @@ public class AsyncMonoRetryTopicScenarioTests { @Autowired DestinationTopicContainer topicContainer; - @KafkaListener( - id = "0-topicId", - topics = TEST_TOPIC0, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener0 { - - @Autowired - CountDownLatchContainer container; - - private final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - @KafkaHandler - public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - return Mono.fromCallable(() -> { - container.extraCountDownLatch0.countDown(); - container.countDownLatch0.countDown(); - try { - Thread.sleep(1); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - throw new RuntimeException("Woooops... in topic " + receivedTopic); - }).then(); - } - - } - @Test - void allFailCaseTest(@Autowired TestTopicListener0 zeroTopicListener, - @Autowired MyCustomDltProcessor myCustomDltProcessor0, - @Autowired ThreadPoolTaskExecutor executor) { + void allFailCaseTest( + @Autowired TestTopicListener0 zeroTopicListener, + @Autowired MyCustomDltProcessor myCustomDltProcessor0, + @Autowired ThreadPoolTaskExecutor executor) { // Given String shortFailedMsg1 = "0"; String shortFailedMsg2 = "1"; @@ -213,43 +179,11 @@ void allFailCaseTest(@Autowired TestTopicListener0 zeroTopicListener, assertThat(latchContainer.extraDltCountdownLatch0.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT0); } - @KafkaListener( - id = "1-topicId", - topics = TEST_TOPIC1, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener1 { - - @Autowired - CountDownLatchContainer container; - - private final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - - @KafkaHandler - public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - - container.extraCountDownLatch1.countDown(); - container.countDownLatch1.countDown(); - return Mono.delay(Duration.ofMillis(Integer.parseInt(message))) // Thread.sleep 대체 - .flatMap(delay -> { - if ("1".equals(message)) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); - }); - } - } - @Test - void firstShortFailAndLastLongSuccessRetryTest(@Autowired TestTopicListener1 testTopicListener1, - @Autowired MyCustomDltProcessor myCustomDltProcessor1, - @Autowired ThreadPoolTaskExecutor executor) { + void firstShortFailAndLastLongSuccessRetryTest( + @Autowired TestTopicListener1 testTopicListener1, + @Autowired MyCustomDltProcessor myCustomDltProcessor1, + @Autowired ThreadPoolTaskExecutor executor) { // Given String longSuccessMsg = "3"; String shortFailedMsg = "1"; @@ -297,42 +231,11 @@ void firstShortFailAndLastLongSuccessRetryTest(@Autowired TestTopicListener1 tes assertThat(latchContainer.extraDltCountdownLatch1.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT1); } - @KafkaListener( - id = "2-topicId", - topics = TEST_TOPIC2, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener2 { - - @Autowired - CountDownLatchContainer container; - - protected final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - @KafkaHandler - public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - - container.extraCountDownLatch2.countDown(); - container.countDownLatch2.countDown(); - return Mono.delay(Duration.ofMillis(Integer.parseInt(message))) // Thread.sleep 대체 - .flatMap(delay -> { - if ("1".equals(message)) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); - }); - } - } - @Test - void firstLongSuccessAndLastShortFailed(@Autowired TestTopicListener2 zero2TopicListener, - @Autowired MyCustomDltProcessor myCustomDltProcessor2, - @Autowired ThreadPoolTaskExecutor executor) { + void firstLongSuccessAndLastShortFailed( + @Autowired TestTopicListener2 zero2TopicListener, + @Autowired MyCustomDltProcessor myCustomDltProcessor2, + @Autowired ThreadPoolTaskExecutor executor) { // Given String shortFailedMsg = "1"; String longSuccessMsg = "3"; @@ -379,67 +282,30 @@ void firstLongSuccessAndLastShortFailed(@Autowired TestTopicListener2 zero2Topic assertThat(latchContainer.extraDltCountdownLatch2.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT2); } - - @KafkaListener( - id = "3-topicId", - topics = TEST_TOPIC3, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener3 { - - @Autowired - CountDownLatchContainer container; - - protected final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - public static final String LONG_FAIL_MSG = "5000"; - public static final String SHORT_SUCCESS_MSG = "1"; - - - @KafkaHandler - public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - - container.extraCountDownLatch3.countDown(); - container.countDownLatch3.countDown(); - return Mono.delay(Duration.ofMillis(Integer.parseInt(message))) // Thread.sleep 대체 - .flatMap(delay -> { - if (message.equals(LONG_FAIL_MSG)) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); - }); - } - } - - @Test - void longFailMsgTwiceThenShortSuccessMsgThird(@Autowired TestTopicListener3 testTopicListener3, - @Autowired MyCustomDltProcessor myCustomDltProcessor3, - @Autowired ThreadPoolTaskExecutor executor) { + void longFailMsgTwiceThenShortSuccessMsgThird( + @Autowired TestTopicListener3 testTopicListener3, + @Autowired MyCustomDltProcessor myCustomDltProcessor3, + @Autowired ThreadPoolTaskExecutor executor) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("3-topicId", TEST_TOPIC3); String expectedRetryTopic = TEST_TOPIC3 + "-retry"; String[] expectedReceivedMsgs = { - TestTopicListener3.LONG_FAIL_MSG, - TestTopicListener3.LONG_FAIL_MSG, - TestTopicListener3.SHORT_SUCCESS_MSG, - TestTopicListener3.SHORT_SUCCESS_MSG, - TestTopicListener3.SHORT_SUCCESS_MSG, - TestTopicListener3.LONG_FAIL_MSG, - TestTopicListener3.LONG_FAIL_MSG, - TestTopicListener3.LONG_FAIL_MSG, - TestTopicListener3.LONG_FAIL_MSG, - TestTopicListener3.LONG_FAIL_MSG, - TestTopicListener3.LONG_FAIL_MSG, - TestTopicListener3.LONG_FAIL_MSG, - TestTopicListener3.LONG_FAIL_MSG, - }; + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.SHORT_SUCCESS_MSG, + TestTopicListener3.SHORT_SUCCESS_MSG, + TestTopicListener3.SHORT_SUCCESS_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + }; String[] expectedReceivedTopics = { TEST_TOPIC3, @@ -458,9 +324,9 @@ void longFailMsgTwiceThenShortSuccessMsgThird(@Autowired TestTopicListener3 test }; String[] expectedDltMsgs = { - TestTopicListener3.LONG_FAIL_MSG, - TestTopicListener3.LONG_FAIL_MSG, - }; + TestTopicListener3.LONG_FAIL_MSG, + TestTopicListener3.LONG_FAIL_MSG, + }; // When kafkaTemplate.send(TEST_TOPIC3, TestTopicListener3.LONG_FAIL_MSG); @@ -483,64 +349,29 @@ void longFailMsgTwiceThenShortSuccessMsgThird(@Autowired TestTopicListener3 test assertThat(latchContainer.extraDltCountdownLatch3.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT3); } - @KafkaListener( - id = "4-TopicId", - topics = TEST_TOPIC4, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener4 { - - @Autowired - CountDownLatchContainer container; - - protected final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - public static final String LONG_SUCCESS_MSG = "5000"; - public static final String SHORT_FAIL_MSG = "1"; - - @KafkaHandler - public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - - container.extraCountDownLatch4.countDown(); - container.countDownLatch4.countDown(); - return Mono.delay(Duration.ofMillis(Integer.parseInt(message))) // Thread.sleep 대체 - .flatMap(delay -> { - if (message.equals(SHORT_FAIL_MSG)) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); - }); - } - } - - @Test - void longSuccessMsgTwiceThenShortFailMsgTwice(@Autowired TestTopicListener4 topicListener4, - @Autowired MyCustomDltProcessor myCustomDltProcessor4, - @Autowired ThreadPoolTaskExecutor executor) { + void longSuccessMsgTwiceThenShortFailMsgTwice( + @Autowired TestTopicListener4 topicListener4, + @Autowired MyCustomDltProcessor myCustomDltProcessor4, + @Autowired ThreadPoolTaskExecutor executor) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("4-TopicId", TEST_TOPIC4); String expectedRetryTopic = TEST_TOPIC4 + "-retry"; String[] expectedReceivedMsgs = { - TestTopicListener4.LONG_SUCCESS_MSG, - TestTopicListener4.LONG_SUCCESS_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - }; + TestTopicListener4.LONG_SUCCESS_MSG, + TestTopicListener4.LONG_SUCCESS_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + }; String[] expectedReceivedTopics = { TEST_TOPIC4, @@ -558,9 +389,9 @@ void longSuccessMsgTwiceThenShortFailMsgTwice(@Autowired TestTopicListener4 topi }; String[] expectedDltMsgs = { - TestTopicListener4.SHORT_FAIL_MSG, - TestTopicListener4.SHORT_FAIL_MSG, - }; + TestTopicListener4.SHORT_FAIL_MSG, + TestTopicListener4.SHORT_FAIL_MSG, + }; // When kafkaTemplate.send(TEST_TOPIC4, TestTopicListener4.LONG_SUCCESS_MSG); @@ -582,48 +413,11 @@ void longSuccessMsgTwiceThenShortFailMsgTwice(@Autowired TestTopicListener4 topi assertThat(latchContainer.extraDltCountdownLatch4.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT4); } - - - @KafkaListener( - id = "5-TopicId", - topics = TEST_TOPIC5, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener5 { - - @Autowired - CountDownLatchContainer container; - - protected final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - public static final String LONG_SUCCESS_MSG = "5000"; - public static final String SHORT_FAIL_MSG = "1"; - - @KafkaHandler - public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - - container.extraCountDownLatch5.countDown(); - container.countDownLatch5.countDown(); - return Mono.delay(Duration.ofMillis(Integer.parseInt(message))) // Thread.sleep 대체 - .flatMap(delay -> { - if (message.startsWith(SHORT_FAIL_MSG)) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); - }); - } - } - - @Test - void oneLongSuccessMsgBetweenHunderedShortFailMsg(@Autowired TestTopicListener5 topicListener5, - @Autowired MyCustomDltProcessor myCustomDltProcessor5, - @Autowired ThreadPoolTaskExecutor executor) { + void oneLongSuccessMsgBetweenHunderedShortFailMsg( + @Autowired TestTopicListener5 topicListener5, + @Autowired MyCustomDltProcessor myCustomDltProcessor5, + @Autowired ThreadPoolTaskExecutor executor) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("5-TopicId", TEST_TOPIC5); @@ -635,7 +429,6 @@ void oneLongSuccessMsgBetweenHunderedShortFailMsg(@Autowired TestTopicListener5 } expectedReceivedMsgs[500] = TestTopicListener5.LONG_SUCCESS_MSG; - String[] expectedReceivedTopics = new String[501]; for (int i = 0; i < 100; i++) { expectedReceivedTopics[i] = TEST_TOPIC5; @@ -673,53 +466,11 @@ void oneLongSuccessMsgBetweenHunderedShortFailMsg(@Autowired TestTopicListener5 assertThat(latchContainer.extraDltCountdownLatch5.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT5); } - - @KafkaListener( - id = "6-TopicId", - topics = TEST_TOPIC6, - containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, - errorHandler = "myCustomErrorHandler", - contentTypeConverter = "myCustomMessageConverter", - concurrency = "2") - static class TestTopicListener6 { - - @Autowired - CountDownLatchContainer container; - - protected final List receivedMsgs = new ArrayList<>(); - private final List receivedTopics = new ArrayList<>(); - - public static final String SUCCESS_SUFFIX = ",s"; - public static final String FAIL_SUFFIX = ",f"; - - @KafkaHandler - public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { - this.receivedMsgs.add(message); - this.receivedTopics.add(receivedTopic); - - - container.extraCountDownLatch6.countDown(); - container.countDownLatch6.countDown(); - - String[] split = message.split(","); - String sleepAWhile = split[0]; - String failOrSuccess = split[1]; - - return Mono.delay(Duration.ofMillis(Integer.parseInt(sleepAWhile))) // Thread.sleep 대체 - .flatMap(delay -> { - if (failOrSuccess.equals("f")) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); - }); - } - } - - @Test - void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime(@Autowired TestTopicListener6 topicListener6, - @Autowired MyCustomDltProcessor myCustomDltProcessor6, - @Autowired ThreadPoolTaskExecutor executor) { + void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime( + @Autowired TestTopicListener6 topicListener6, + @Autowired MyCustomDltProcessor myCustomDltProcessor6, + @Autowired ThreadPoolTaskExecutor executor) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("6-TopicId", TEST_TOPIC6); @@ -770,17 +521,17 @@ void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime(@Autowired TestTopicListe long actualReceivedFailedMsgCount = topicListener6.receivedMsgs.stream() .map(s -> s.split(",")[1]) .filter(m -> (',' + m).equals( - TestTopicListener6.FAIL_SUFFIX)) + TestTopicListener6.FAIL_SUFFIX)) .count(); long actualReceivedOriginalTopicMsgCount = topicListener6.receivedTopics.stream() - .filter(topic -> topic.equals(TEST_TOPIC6)) - .count(); + .filter(topic -> topic.equals(TEST_TOPIC6)) + .count(); long actualReceivedRetryTopicMsgCount = topicListener6.receivedTopics.stream() - .filter(topic -> topic.equals(expectedRetryTopic)) - .count(); + .filter(topic -> topic.equals(expectedRetryTopic)) + .count(); assertThat(actualReceivedSuccessMsgCount).isEqualTo(expectedSuccessMsgCount); assertThat(actualReceivedFailedMsgCount).isEqualTo(expectedFailedMsgCount); @@ -791,7 +542,295 @@ void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime(@Autowired TestTopicListe assertThat(latchContainer.extraDltCountdownLatch6.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT6); } + private boolean awaitLatch(CountDownLatch latch) { + try { + return latch.await(60, TimeUnit.SECONDS); + } + catch (Exception e) { + fail(e.getMessage()); + throw new RuntimeException(e); + } + } + + private void waitAWhile(ThreadPoolTaskExecutor executor, int sleep) { + CountDownLatch countDownLatch = new CountDownLatch(1); + + Thread thread = new Thread(() -> { + try { + Thread.sleep(sleep); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + countDownLatch.countDown(); + } + ); + executor.execute(thread); + assertThat(awaitLatch(countDownLatch)).isTrue(); + } + + @KafkaListener( + id = "0-topicId", + topics = TEST_TOPIC0, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener0 { + + @Autowired + CountDownLatchContainer container; + + private final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + return Mono.fromCallable(() -> { + container.extraCountDownLatch0.countDown(); + container.countDownLatch0.countDown(); + try { + Thread.sleep(1); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("Woooops... in topic " + receivedTopic); + }).then(); + } + + } + + @KafkaListener( + id = "1-topicId", + topics = TEST_TOPIC1, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener1 { + + @Autowired + CountDownLatchContainer container; + + private final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch1.countDown(); + container.countDownLatch1.countDown(); + return Mono + .delay(Duration.ofMillis(Integer.parseInt(message))) + .flatMap(delay -> { + if ("1".equals(message)) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + }); + } + } + + @KafkaListener( + id = "2-topicId", + topics = TEST_TOPIC2, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener2 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch2.countDown(); + container.countDownLatch2.countDown(); + return Mono + .delay(Duration.ofMillis(Integer.parseInt(message))) + .flatMap(delay -> { + if ("1".equals(message)) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + }); + } + } + + @KafkaListener( + id = "3-topicId", + topics = TEST_TOPIC3, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener3 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + public static final String LONG_FAIL_MSG = "5000"; + + public static final String SHORT_SUCCESS_MSG = "1"; + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch3.countDown(); + container.countDownLatch3.countDown(); + return Mono + .delay(Duration.ofMillis(Integer.parseInt(message))) + .flatMap(delay -> { + if (message.equals(LONG_FAIL_MSG)) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + }); + } + } + + @KafkaListener( + id = "4-TopicId", + topics = TEST_TOPIC4, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener4 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + public static final String LONG_SUCCESS_MSG = "5000"; + + public static final String SHORT_FAIL_MSG = "1"; + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch4.countDown(); + container.countDownLatch4.countDown(); + return Mono + .delay(Duration.ofMillis(Integer.parseInt(message))) + .flatMap(delay -> { + if (message.equals(SHORT_FAIL_MSG)) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + }); + } + } + + @KafkaListener( + id = "5-TopicId", + topics = TEST_TOPIC5, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener5 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + public static final String LONG_SUCCESS_MSG = "5000"; + + public static final String SHORT_FAIL_MSG = "1"; + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch5.countDown(); + container.countDownLatch5.countDown(); + return Mono + .delay(Duration.ofMillis(Integer.parseInt(message))) + .flatMap(delay -> { + if (message.startsWith(SHORT_FAIL_MSG)) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + } + ); + } + } + + @KafkaListener( + id = "6-TopicId", + topics = TEST_TOPIC6, + containerFactory = MAIN_TOPIC_CONTAINER_FACTORY, + errorHandler = "myCustomErrorHandler", + contentTypeConverter = "myCustomMessageConverter", + concurrency = "2") + static class TestTopicListener6 { + + @Autowired + CountDownLatchContainer container; + + protected final List receivedMsgs = new ArrayList<>(); + + private final List receivedTopics = new ArrayList<>(); + + public static final String SUCCESS_SUFFIX = ",s"; + + public static final String FAIL_SUFFIX = ",f"; + + @KafkaHandler + public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + this.receivedMsgs.add(message); + this.receivedTopics.add(receivedTopic); + + container.extraCountDownLatch6.countDown(); + container.countDownLatch6.countDown(); + + String[] split = message.split(","); + String sleepAWhile = split[0]; + String failOrSuccess = split[1]; + + return Mono + .delay(Duration.ofMillis(Integer.parseInt(sleepAWhile))) + .flatMap(delay -> { + if (failOrSuccess.equals("f")) { + return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); + } + return Mono.just("Task Completed"); + } + ); + } + } static class CountDownLatchContainer { @@ -843,7 +882,6 @@ static class CountDownLatchContainer { CountDownLatch extraDltCountdownLatch3 = new CountDownLatch(1000); - static int COUNT4 = 12; static int DLT_COUNT4 = 2; @@ -856,7 +894,6 @@ static class CountDownLatchContainer { CountDownLatch extraDltCountdownLatch4 = new CountDownLatch(1000); - static int COUNT5 = 501; static int DLT_COUNT5 = 100; @@ -869,7 +906,6 @@ static class CountDownLatchContainer { CountDownLatch extraDltCountdownLatch5 = new CountDownLatch(1000); - static int COUNT6 = 250; static int DLT_COUNT6 = 50; @@ -892,7 +928,7 @@ static class MyCustomDltProcessor { final List receivedMsg = new ArrayList<>(); - public MyCustomDltProcessor(KafkaTemplate kafkaTemplate, + MyCustomDltProcessor(KafkaTemplate kafkaTemplate, CountDownLatch latch, CountDownLatch extraLatch) { this.kafkaTemplate = kafkaTemplate; @@ -949,54 +985,60 @@ static RetryTopicConfiguration createRetryTopicConfiguration(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC0, - "myCustomDltProcessor0"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC0, + "myCustomDltProcessor0"); } @Bean RetryTopicConfiguration testRetryTopic1(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC1, - "myCustomDltProcessor1"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC1, + "myCustomDltProcessor1"); } @Bean RetryTopicConfiguration testRetryTopic2(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC2, - "myCustomDltProcessor2"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC2, + "myCustomDltProcessor2"); } @Bean RetryTopicConfiguration testRetryTopic3(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC3, - "myCustomDltProcessor3"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC3, + "myCustomDltProcessor3"); } @Bean RetryTopicConfiguration testRetryTopic4(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC4, - "myCustomDltProcessor4"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC4, + "myCustomDltProcessor4"); } @Bean RetryTopicConfiguration testRetryTopic5(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC5, - "myCustomDltProcessor5"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC5, + "myCustomDltProcessor5"); } @Bean RetryTopicConfiguration testRetryTopic6(KafkaTemplate template) { - return createRetryTopicConfiguration(template, - TEST_TOPIC6, - "myCustomDltProcessor6"); + return createRetryTopicConfiguration( + template, + TEST_TOPIC6, + "myCustomDltProcessor6"); } - @Bean KafkaListenerErrorHandler myCustomErrorHandler( CountDownLatchContainer container) { @@ -1060,56 +1102,63 @@ TestTopicListener6 testTopicListener6() { } @Bean - MyCustomDltProcessor myCustomDltProcessor0(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor0( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch0, latchContainer.extraDltCountdownLatch0); } @Bean - MyCustomDltProcessor myCustomDltProcessor1(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor1( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch1, latchContainer.extraDltCountdownLatch1); } @Bean - MyCustomDltProcessor myCustomDltProcessor2(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor2( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch2, latchContainer.extraDltCountdownLatch2); } @Bean - MyCustomDltProcessor myCustomDltProcessor3(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor3( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch3, latchContainer.extraDltCountdownLatch3); } @Bean - MyCustomDltProcessor myCustomDltProcessor4(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor4( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch4, latchContainer.extraDltCountdownLatch4); } @Bean - MyCustomDltProcessor myCustomDltProcessor5(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor5( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch5, latchContainer.extraDltCountdownLatch5); } @Bean - MyCustomDltProcessor myCustomDltProcessor6(KafkaTemplate kafkaTemplate, - CountDownLatchContainer latchContainer) { + MyCustomDltProcessor myCustomDltProcessor6( + KafkaTemplate kafkaTemplate, + CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, latchContainer.dltCountdownLatch6, latchContainer.extraDltCountdownLatch6); @@ -1158,7 +1207,6 @@ KafkaAdmin kafkaAdmin() { return new KafkaAdmin(configs); } - @Bean ConsumerFactory consumerFactory() { Map props = new HashMap<>(); @@ -1225,30 +1273,4 @@ ThreadPoolTaskExecutor threadPoolTaskExecutor() { } - private boolean awaitLatch(CountDownLatch latch) { - try { - return latch.await(60, TimeUnit.SECONDS); - } - catch (Exception e) { - fail(e.getMessage()); - throw new RuntimeException(e); - } - } - - private void waitAWhile(ThreadPoolTaskExecutor executor, int sleep) { - CountDownLatch countDownLatch = new CountDownLatch(1); - - Thread thread = new Thread(() -> { - try { - Thread.sleep(sleep); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - countDownLatch.countDown(); - }); - - executor.execute(thread); - assertThat(awaitLatch(countDownLatch)).isTrue(); - } - } From 6399b81e52869b777e21a3493460f05893072707 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Wed, 9 Oct 2024 09:12:38 +0900 Subject: [PATCH 07/30] Minimize FailedRecordTuple dependency. --- .../kafka/core/FailedRecordTuple.java | 32 ------------------- .../KafkaMessageListenerContainer.java | 10 ++++-- .../kafka/listener/MessageListener.java | 7 ++-- ...fkaBackoffAwareMessageListenerAdapter.java | 4 +-- .../MessagingMessageListenerAdapter.java | 10 ++---- ...RecordMessagingMessageListenerAdapter.java | 4 +-- 6 files changed, 17 insertions(+), 50 deletions(-) delete mode 100644 spring-kafka/src/main/java/org/springframework/kafka/core/FailedRecordTuple.java diff --git a/spring-kafka/src/main/java/org/springframework/kafka/core/FailedRecordTuple.java b/spring-kafka/src/main/java/org/springframework/kafka/core/FailedRecordTuple.java deleted file mode 100644 index acf9547708..0000000000 --- a/spring-kafka/src/main/java/org/springframework/kafka/core/FailedRecordTuple.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2016-2024 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * ... - * @author Sanghyeok An - * - * @since 3.3 - */ - -package org.springframework.kafka.core; - -import org.apache.kafka.clients.consumer.ConsumerRecord; - -public record FailedRecordTuple( - ConsumerRecord record, - RuntimeException ex) { - -}; diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index a2caae5b56..0826829733 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -84,7 +84,6 @@ import org.springframework.core.task.SimpleAsyncTaskExecutor; import org.springframework.kafka.KafkaException; import org.springframework.kafka.core.ConsumerFactory; -import org.springframework.kafka.core.FailedRecordTuple; import org.springframework.kafka.core.KafkaAdmin; import org.springframework.kafka.core.KafkaResourceHolder; import org.springframework.kafka.event.ConsumerFailedToStartEvent; @@ -902,8 +901,11 @@ else if (listener instanceof MessageListener) { this.observationEnabled = this.containerProperties.isObservationEnabled(); if (!AopUtils.isAopProxy(listener)) { - final java.util.function.Consumer> callbackForAsyncFailureQueue = - (fRecord) -> this.failedRecords.addLast(fRecord); + BiConsumer, RuntimeException> callbackForAsyncFailureQueue = + (cRecord, ex) -> { + FailedRecordTuple failedRecord = new FailedRecordTuple<>(cRecord, ex); + this.failedRecords.addLast(failedRecord); + }; this.listener.setCallbackForAsyncFailureQueue(callbackForAsyncFailureQueue); } } @@ -3983,4 +3985,6 @@ private static class StopAfterFenceException extends KafkaException { } + record FailedRecordTuple(ConsumerRecord record, RuntimeException ex) { }; + } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java index 36d0389510..e00eebb15b 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java @@ -16,12 +16,10 @@ package org.springframework.kafka.listener; -import java.util.function.Consumer; +import java.util.function.BiConsumer; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.springframework.kafka.core.FailedRecordTuple; - /** * Listener for handling individual incoming Kafka messages. * @@ -35,7 +33,8 @@ @FunctionalInterface public interface MessageListener extends GenericMessageListener> { - default void setCallbackForAsyncFailureQueue(Consumer> asyncRetryCallback) { + default void setCallbackForAsyncFailureQueue( + BiConsumer, RuntimeException> asyncRetryCallback) { // } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java index 9a28ec3ebe..d35d8f0831 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java @@ -20,12 +20,12 @@ import java.time.Clock; import java.time.Instant; import java.util.Optional; +import java.util.function.BiConsumer; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.TopicPartition; -import org.springframework.kafka.core.FailedRecordTuple; import org.springframework.kafka.listener.AcknowledgingConsumerAwareMessageListener; import org.springframework.kafka.listener.KafkaBackoffException; import org.springframework.kafka.listener.KafkaConsumerBackoffManager; @@ -147,7 +147,7 @@ public void onMessage(ConsumerRecord data, Consumer consumer) { } @Override - public void setCallbackForAsyncFailureQueue(java.util.function.Consumer> callbackForAsyncFailureQueue) { + public void setCallbackForAsyncFailureQueue(BiConsumer, RuntimeException> callbackForAsyncFailureQueue) { this.delegate.setCallbackForAsyncFailureQueue(callbackForAsyncFailureQueue); } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index ec84651755..7b81d0d4a3 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -47,7 +47,6 @@ import org.springframework.expression.spel.standard.SpelExpressionParser; import org.springframework.expression.spel.support.StandardEvaluationContext; import org.springframework.expression.spel.support.StandardTypeConverter; -import org.springframework.kafka.core.FailedRecordTuple; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.listener.ConsumerSeekAware; import org.springframework.kafka.listener.KafkaListenerErrorHandler; @@ -156,9 +155,7 @@ public abstract class MessagingMessageListenerAdapter implements ConsumerS private String correlationHeaderName = KafkaHeaders.CORRELATION_ID; - private BiConsumer, RuntimeException> asyncRetryCallback; - - private java.util.function.Consumer> callbackForAsyncFailureQueue; + private BiConsumer, RuntimeException> callbackForAsyncFailureQueue; /** * Create an instance with the provided bean and method. @@ -691,8 +688,7 @@ protected void asyncFailure(Object request, @Nullable Acknowledgment acknowledgm acknowledge(acknowledgment); if (canAsyncRetry(request, ex)) { ConsumerRecord record = (ConsumerRecord) request; - FailedRecordTuple failedRecordTuple = new FailedRecordTuple<>(record, (RuntimeException) ex); - this.callbackForAsyncFailureQueue.accept(failedRecordTuple); + this.callbackForAsyncFailureQueue.accept(record, (RuntimeException) ex); } } } @@ -896,7 +892,7 @@ private boolean rawByParameterIsType(Type parameterType, Type type) { return parameterType instanceof ParameterizedType pType && pType.getRawType().equals(type); } - public void putInAsyncFailureQueue(java.util.function.Consumer> callbackForAsyncFailureQueue) { + public void putInAsyncFailureQueue(BiConsumer, RuntimeException> callbackForAsyncFailureQueue) { this.callbackForAsyncFailureQueue = callbackForAsyncFailureQueue; } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java index e006eb051f..209a4c7286 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java @@ -17,11 +17,11 @@ package org.springframework.kafka.listener.adapter; import java.lang.reflect.Method; +import java.util.function.BiConsumer; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.springframework.kafka.core.FailedRecordTuple; import org.springframework.kafka.listener.AcknowledgingConsumerAwareMessageListener; import org.springframework.kafka.listener.KafkaListenerErrorHandler; import org.springframework.kafka.support.Acknowledgment; @@ -88,7 +88,7 @@ public void onMessage(ConsumerRecord record, @Nullable Acknowledgment ackn @Override public void setCallbackForAsyncFailureQueue( - java.util.function.Consumer> asyncRetryCallback) { + BiConsumer, RuntimeException> asyncRetryCallback) { putInAsyncFailureQueue(asyncRetryCallback); } } From 4a4a1ee601195b9ae5680583c4548eedb662f6c2 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Thu, 10 Oct 2024 09:08:07 +0900 Subject: [PATCH 08/30] Refactor test codes and real codes. --- .../KafkaMessageListenerContainer.java | 7 ++- .../MessagingMessageListenerAdapter.java | 16 ++--- ...eRetryTopicClassLevelIntegrationTests.java | 44 ++++---------- ...pletableFutureRetryTopicScenarioTests.java | 58 ++++--------------- ...eRetryTopicClassLevelIntegrationTests.java | 40 +++---------- .../AsyncMonoRetryTopicScenarioTests.java | 51 +++------------- 6 files changed, 48 insertions(+), 168 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index 0826829733..03adedfec0 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -1312,9 +1312,8 @@ public void run() { handleAsyncFailure(); } catch (Exception e) { - // TODO: Need to improve error handling. - // TODO: Need to determine how to handle a failed message. - this.logger.error("Failed to process re-try messages. "); + ListenerConsumer.this.logger.error( + "Failed to process async retry messages. skip this time, try it again next loop."); } try { @@ -1465,6 +1464,8 @@ protected void handleAsyncFailure() { copyFailedRecords.add(failedRecordTuple); } + // If any copied and failed record fails to complete due to an unexpected error, + // We will give up on retrying with the remaining copied and failed Records. if (!copyFailedRecords.isEmpty()) { copyFailedRecords.forEach(failedRecordTuple -> this.invokeErrorHandlerBySingleRecord(failedRecordTuple.record(), failedRecordTuple.ex())); diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index 7b81d0d4a3..e439ea9dfb 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -670,18 +670,10 @@ protected void asyncFailure(Object request, @Nullable Acknowledgment acknowledgm Throwable t, Message source) { try { - if (t instanceof CompletionException) { - // For CompletableFuture Object - handleException(request, acknowledgment, consumer, source, - new ListenerExecutionFailedException(createMessagingErrorMessage( - "Async Fail", source.getPayload()), t.getCause())); - } - else { - // For Mono Object. - handleException(request, acknowledgment, consumer, source, - new ListenerExecutionFailedException(createMessagingErrorMessage( - "Async Fail", source.getPayload()), t)); - } + Throwable cause = t instanceof CompletionException ? t.getCause() : t; + handleException(request, acknowledgment, consumer, source, + new ListenerExecutionFailedException(createMessagingErrorMessage( + "Async Fail", source.getPayload()), cause)); } catch (Throwable ex) { this.logger.error(t, () -> "Future, Mono, or suspend function was completed with an exception for " + source); diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java index d5b2d301b5..0a6c64ecf5 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java @@ -75,6 +75,7 @@ import org.springframework.kafka.support.KafkaHeaders; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.kafka.test.utils.KafkaTestUtils; import org.springframework.messaging.Message; import org.springframework.messaging.converter.CompositeMessageConverter; import org.springframework.messaging.converter.GenericMessageConverter; @@ -86,7 +87,7 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; - +import org.springframework.util.ReflectionUtils; /** * @author Sanghyeok An @@ -95,13 +96,7 @@ @SpringJUnitConfig @DirtiesContext -@EmbeddedKafka(topics = { - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.FIRST_TOPIC, - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.SECOND_TOPIC, - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.THIRD_TOPIC, - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.FOURTH_TOPIC, - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.TWO_LISTENERS_TOPIC, - AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.MANUAL_TOPIC }) +@EmbeddedKafka @TestPropertySource(properties = { "five.attempts=5", "kafka.template=customKafkaTemplate"}) public class AsyncCompletableFutureRetryTopicClassLevelIntegrationTests { @@ -181,7 +176,7 @@ void shouldRetryThirdTopicWithTimeout( assertThat(topics.get(THIRD_TOPIC + "-dlt").partitions()).hasSize(3); assertThat(topics.get(FOURTH_TOPIC).partitions()).hasSize(2); AtomicReference method = new AtomicReference<>(); - org.springframework.util.ReflectionUtils.doWithMethods(KafkaAdmin.class, m -> { + ReflectionUtils.doWithMethods(KafkaAdmin.class, m -> { m.setAccessible(true); method.set(m); }, m -> m.getName().equals("newTopics")); @@ -881,17 +876,10 @@ static class KafkaProducerConfig { @Bean ProducerFactory producerFactory() { - Map configProps = new HashMap<>(); - configProps.put( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + Map props = KafkaTestUtils.producerProps( this.broker.getBrokersAsString()); - configProps.put( - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - configProps.put( - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - return new DefaultKafkaProducerFactory<>(configProps); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + return new DefaultKafkaProducerFactory<>(props); } @Bean("customKafkaTemplate") @@ -926,23 +914,13 @@ NewTopics topics() { @Bean ConsumerFactory consumerFactory() { - Map props = new HashMap<>(); - props.put( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - this.broker.getBrokersAsString()); - props.put( - ConsumerConfig.GROUP_ID_CONFIG, - "groupId"); + Map props = KafkaTestUtils.consumerProps( + this.broker.getBrokersAsString(), + "groupId", + "false"); props.put( ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - props.put( - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class); - props.put( - ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - return new DefaultKafkaConsumerFactory<>(props); } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java index 98c46c2eb2..5b949b18f9 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -31,9 +31,7 @@ import java.util.concurrent.TimeUnit; import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.Test; @@ -56,6 +54,7 @@ import org.springframework.kafka.support.KafkaHeaders; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.kafka.test.utils.KafkaTestUtils; import org.springframework.messaging.Message; import org.springframework.messaging.converter.CompositeMessageConverter; import org.springframework.messaging.converter.GenericMessageConverter; @@ -979,28 +978,15 @@ public void processDltMessage(String message) { } } - @SuppressWarnings("serial") - static class MyRetryException extends RuntimeException { - MyRetryException(String msg) { - super(msg); - } - } - - @SuppressWarnings("serial") - static class MyDontRetryException extends RuntimeException { - MyDontRetryException(String msg) { - super(msg); - } - } - @Configuration static class RetryTopicConfigurations extends RetryTopicConfigurationSupport { private static final String DLT_METHOD_NAME = "processDltMessage"; - static RetryTopicConfiguration createRetryTopicConfiguration(KafkaTemplate template, - String topicName, - String dltBeanName) { + static RetryTopicConfiguration createRetryTopicConfiguration( + KafkaTemplate template, + String topicName, + String dltBeanName) { return RetryTopicConfigurationBuilder .newInstance() .fixedBackOff(50) @@ -1204,17 +1190,10 @@ static class KafkaProducerConfig { @Bean ProducerFactory producerFactory() { - Map configProps = new HashMap<>(); - configProps.put( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + Map props = KafkaTestUtils.producerProps( this.broker.getBrokersAsString()); - configProps.put( - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - configProps.put( - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - return new DefaultKafkaProducerFactory<>(configProps); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + return new DefaultKafkaProducerFactory<>(props); } @Bean("customKafkaTemplate") @@ -1240,23 +1219,10 @@ KafkaAdmin kafkaAdmin() { @Bean ConsumerFactory consumerFactory() { - Map props = new HashMap<>(); - props.put( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - this.broker.getBrokersAsString()); - props.put( - ConsumerConfig.GROUP_ID_CONFIG, - "groupId"); - props.put( - ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class); - props.put( - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class); - props.put( - ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - + Map props = KafkaTestUtils.consumerProps( + this.broker.getBrokersAsString(), + "groupId", + "false"); return new DefaultKafkaConsumerFactory<>(props); } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java index 31bb05b930..4eca91f805 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java @@ -74,6 +74,7 @@ import org.springframework.kafka.support.KafkaHeaders; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.kafka.test.utils.KafkaTestUtils; import org.springframework.messaging.Message; import org.springframework.messaging.converter.CompositeMessageConverter; import org.springframework.messaging.converter.GenericMessageConverter; @@ -95,13 +96,7 @@ @SpringJUnitConfig @DirtiesContext -@EmbeddedKafka(topics = { - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.FIRST_TOPIC, - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.SECOND_TOPIC, - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.THIRD_TOPIC, - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.FOURTH_TOPIC, - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.TWO_LISTENERS_TOPIC, - AsyncMonoFutureRetryTopicClassLevelIntegrationTests.MANUAL_TOPIC }) +@EmbeddedKafka @TestPropertySource(properties = { "five.attempts=5", "kafka.template=customKafkaTemplate"}) public class AsyncMonoFutureRetryTopicClassLevelIntegrationTests { @@ -886,17 +881,10 @@ static class KafkaProducerConfig { @Bean ProducerFactory producerFactory() { - Map configProps = new HashMap<>(); - configProps.put( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + Map props = KafkaTestUtils.producerProps( this.broker.getBrokersAsString()); - configProps.put( - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - configProps.put( - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - return new DefaultKafkaProducerFactory<>(configProps); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + return new DefaultKafkaProducerFactory<>(props); } @Bean("customKafkaTemplate") @@ -931,23 +919,13 @@ NewTopics topics() { @Bean ConsumerFactory consumerFactory() { - Map props = new HashMap<>(); - props.put( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - this.broker.getBrokersAsString()); - props.put( - ConsumerConfig.GROUP_ID_CONFIG, - "groupId"); + Map props = KafkaTestUtils.consumerProps( + this.broker.getBrokersAsString(), + "groupId", + "false"); props.put( ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - props.put( - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class); - props.put( - ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - return new DefaultKafkaConsumerFactory<>(props); } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java index 49f67beab7..7426cdfde8 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -31,9 +31,7 @@ import java.util.concurrent.TimeUnit; import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.Test; @@ -56,6 +54,7 @@ import org.springframework.kafka.support.KafkaHeaders; import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.kafka.test.utils.KafkaTestUtils; import org.springframework.messaging.Message; import org.springframework.messaging.converter.CompositeMessageConverter; import org.springframework.messaging.converter.GenericMessageConverter; @@ -949,20 +948,6 @@ public void processDltMessage(String message) { } } - @SuppressWarnings("serial") - static class MyRetryException extends RuntimeException { - MyRetryException(String msg) { - super(msg); - } - } - - @SuppressWarnings("serial") - static class MyDontRetryException extends RuntimeException { - MyDontRetryException(String msg) { - super(msg); - } - } - @Configuration static class RetryTopicConfigurations extends RetryTopicConfigurationSupport { @@ -1174,17 +1159,10 @@ static class KafkaProducerConfig { @Bean ProducerFactory producerFactory() { - Map configProps = new HashMap<>(); - configProps.put( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + Map props = KafkaTestUtils.producerProps( this.broker.getBrokersAsString()); - configProps.put( - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - configProps.put( - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - return new DefaultKafkaProducerFactory<>(configProps); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + return new DefaultKafkaProducerFactory<>(props); } @Bean("customKafkaTemplate") @@ -1209,23 +1187,10 @@ KafkaAdmin kafkaAdmin() { @Bean ConsumerFactory consumerFactory() { - Map props = new HashMap<>(); - props.put( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - this.broker.getBrokersAsString()); - props.put( - ConsumerConfig.GROUP_ID_CONFIG, - "groupId"); - props.put( - ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class); - props.put( - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class); - props.put( - ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - + Map props = KafkaTestUtils.consumerProps( + this.broker.getBrokersAsString(), + "groupId", + "false"); return new DefaultKafkaConsumerFactory<>(props); } From 08e7721cd5a8aa6d9308ebedf1aa338589517d26 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Thu, 10 Oct 2024 21:11:45 +0900 Subject: [PATCH 09/30] Modify method name. --- .../kafka/listener/KafkaMessageListenerContainer.java | 2 +- .../org/springframework/kafka/listener/MessageListener.java | 2 +- .../adapter/KafkaBackoffAwareMessageListenerAdapter.java | 4 ++-- .../listener/adapter/MessagingMessageListenerAdapter.java | 4 ++-- .../adapter/RecordMessagingMessageListenerAdapter.java | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index 03adedfec0..953dc943cb 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -906,7 +906,7 @@ else if (listener instanceof MessageListener) { FailedRecordTuple failedRecord = new FailedRecordTuple<>(cRecord, ex); this.failedRecords.addLast(failedRecord); }; - this.listener.setCallbackForAsyncFailureQueue(callbackForAsyncFailureQueue); + this.listener.setCallbackForAsyncFailure(callbackForAsyncFailureQueue); } } else { diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java index e00eebb15b..1976efa226 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java @@ -33,7 +33,7 @@ @FunctionalInterface public interface MessageListener extends GenericMessageListener> { - default void setCallbackForAsyncFailureQueue( + default void setCallbackForAsyncFailure( BiConsumer, RuntimeException> asyncRetryCallback) { // } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java index d35d8f0831..a2774a8f4e 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java @@ -147,7 +147,7 @@ public void onMessage(ConsumerRecord data, Consumer consumer) { } @Override - public void setCallbackForAsyncFailureQueue(BiConsumer, RuntimeException> callbackForAsyncFailureQueue) { - this.delegate.setCallbackForAsyncFailureQueue(callbackForAsyncFailureQueue); + public void setCallbackForAsyncFailure(BiConsumer, RuntimeException> callbackForAsyncFailureQueue) { + this.delegate.setCallbackForAsyncFailure(callbackForAsyncFailureQueue); } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index e439ea9dfb..5acdf94956 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -884,8 +884,8 @@ private boolean rawByParameterIsType(Type parameterType, Type type) { return parameterType instanceof ParameterizedType pType && pType.getRawType().equals(type); } - public void putInAsyncFailureQueue(BiConsumer, RuntimeException> callbackForAsyncFailureQueue) { - this.callbackForAsyncFailureQueue = callbackForAsyncFailureQueue; + protected void setCallbackForAsyncFailure(BiConsumer, RuntimeException> asyncRetryCallback) { + this.callbackForAsyncFailureQueue = asyncRetryCallback; } /** diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java index 209a4c7286..9904d1575e 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java @@ -87,8 +87,8 @@ public void onMessage(ConsumerRecord record, @Nullable Acknowledgment ackn } @Override - public void setCallbackForAsyncFailureQueue( + public void setCallbackForAsyncFailure( BiConsumer, RuntimeException> asyncRetryCallback) { - putInAsyncFailureQueue(asyncRetryCallback); + super.setCallbackForAsyncFailure(asyncRetryCallback); } } From c0fea39a8accf8dadd082fc9bac3e6dc94686c92 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Thu, 10 Oct 2024 21:25:20 +0900 Subject: [PATCH 10/30] Fixes compile error. --- .../kafka/listener/adapter/MessagingMessageListenerAdapter.java | 2 +- .../listener/adapter/RecordMessagingMessageListenerAdapter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index 5acdf94956..3e63bd0ec5 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -884,7 +884,7 @@ private boolean rawByParameterIsType(Type parameterType, Type type) { return parameterType instanceof ParameterizedType pType && pType.getRawType().equals(type); } - protected void setCallbackForAsyncFailure(BiConsumer, RuntimeException> asyncRetryCallback) { + protected void setAsyncFailureCallback(BiConsumer, RuntimeException> asyncRetryCallback) { this.callbackForAsyncFailureQueue = asyncRetryCallback; } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java index 9904d1575e..ccfcb52b9a 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java @@ -89,6 +89,6 @@ public void onMessage(ConsumerRecord record, @Nullable Acknowledgment ackn @Override public void setCallbackForAsyncFailure( BiConsumer, RuntimeException> asyncRetryCallback) { - super.setCallbackForAsyncFailure(asyncRetryCallback); + setAsyncFailureCallback(asyncRetryCallback); } } From e74072c6a95630369b5a13ba012928e023cf24d2 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Fri, 11 Oct 2024 19:03:01 +0900 Subject: [PATCH 11/30] Remove async retry callback from MessageListener contract. --- .../KafkaMessageListenerContainer.java | 49 +++++++++++++------ .../kafka/listener/MessageListener.java | 7 --- ...fkaBackoffAwareMessageListenerAdapter.java | 6 --- .../MessagingMessageListenerAdapter.java | 12 +++-- ...RecordMessagingMessageListenerAdapter.java | 6 --- ...eRetryTopicClassLevelIntegrationTests.java | 2 +- ...pletableFutureRetryTopicScenarioTests.java | 12 +---- ...eRetryTopicClassLevelIntegrationTests.java | 2 +- .../AsyncMonoRetryTopicScenarioTests.java | 12 +---- 9 files changed, 45 insertions(+), 63 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index 953dc943cb..957f64c4c9 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -108,6 +108,8 @@ import org.springframework.kafka.listener.ContainerProperties.AssignmentCommitOption; import org.springframework.kafka.listener.ContainerProperties.EOSMode; import org.springframework.kafka.listener.adapter.AsyncRepliesAware; +import org.springframework.kafka.listener.adapter.KafkaBackoffAwareMessageListenerAdapter; +import org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter; import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.KafkaHeaders; import org.springframework.kafka.support.KafkaUtils; @@ -842,7 +844,7 @@ private final class ListenerConsumer implements SchedulingAwareRunnable, Consume private volatile long lastPoll = System.currentTimeMillis(); - private final ConcurrentLinkedDeque> failedRecords = new ConcurrentLinkedDeque(); + private final ConcurrentLinkedDeque> failedRecords = new ConcurrentLinkedDeque<>(); @SuppressWarnings(UNCHECKED) ListenerConsumer(GenericMessageListener listener, ListenerType listenerType, @@ -900,13 +902,20 @@ else if (listener instanceof MessageListener) { this.pollThreadStateProcessor = setUpPollProcessor(false); this.observationEnabled = this.containerProperties.isObservationEnabled(); - if (!AopUtils.isAopProxy(listener)) { - BiConsumer, RuntimeException> callbackForAsyncFailureQueue = - (cRecord, ex) -> { - FailedRecordTuple failedRecord = new FailedRecordTuple<>(cRecord, ex); - this.failedRecords.addLast(failedRecord); - }; - this.listener.setCallbackForAsyncFailure(callbackForAsyncFailureQueue); + if (!AopUtils.isAopProxy(this.genericListener) && + this.genericListener instanceof KafkaBackoffAwareMessageListenerAdapter) { + KafkaBackoffAwareMessageListenerAdapter genListener = + (KafkaBackoffAwareMessageListenerAdapter) this.genericListener; + if (genListener.getDelegate() instanceof RecordMessagingMessageListenerAdapter) { + + RecordMessagingMessageListenerAdapter recordAdapterListener = + (RecordMessagingMessageListenerAdapter) genListener.getDelegate(); + + BiConsumer, RuntimeException> callbackForAsyncFailure = + (cRecord, ex) -> this.failedRecords.addLast(new FailedRecordTuple<>(cRecord, ex)); + recordAdapterListener.setCallbackForAsyncFailure(callbackForAsyncFailure); + } + } } else { @@ -1458,17 +1467,25 @@ protected void pollAndInvoke() { } protected void handleAsyncFailure() { - List copyFailedRecords = new ArrayList<>(); + List> copyFailedRecords = new ArrayList<>(); while (!this.failedRecords.isEmpty()) { - FailedRecordTuple failedRecordTuple = this.failedRecords.pollFirst(); + FailedRecordTuple failedRecordTuple = this.failedRecords.pollFirst(); copyFailedRecords.add(failedRecordTuple); } // If any copied and failed record fails to complete due to an unexpected error, // We will give up on retrying with the remaining copied and failed Records. - if (!copyFailedRecords.isEmpty()) { - copyFailedRecords.forEach(failedRecordTuple -> - this.invokeErrorHandlerBySingleRecord(failedRecordTuple.record(), failedRecordTuple.ex())); + for (FailedRecordTuple copyFailedRecord : copyFailedRecords) { + try { + invokeErrorHandlerBySingleRecord(copyFailedRecord); + } + catch (Exception e) { + this.logger.warn(() -> + "Async failed record failed to complete, thus skip it. record :" + + copyFailedRecord.toString() + + ", Exception : " + + e.getMessage()); + } } } @@ -2864,7 +2881,9 @@ private void doInvokeOnMessage(final ConsumerRecord recordArg) { } } - private void invokeErrorHandlerBySingleRecord(final ConsumerRecord cRecord, RuntimeException rte) { + private void invokeErrorHandlerBySingleRecord(final FailedRecordTuple failedRecordTuple) { + final ConsumerRecord cRecord = failedRecordTuple.record; + RuntimeException rte = failedRecordTuple.ex; if (this.commonErrorHandler.seeksAfterHandling() || rte instanceof CommitFailedException) { try { if (this.producer == null) { @@ -3986,6 +4005,6 @@ private static class StopAfterFenceException extends KafkaException { } - record FailedRecordTuple(ConsumerRecord record, RuntimeException ex) { }; + private record FailedRecordTuple(ConsumerRecord record, RuntimeException ex) { }; } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java index 1976efa226..5b20488d65 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java @@ -16,8 +16,6 @@ package org.springframework.kafka.listener; -import java.util.function.BiConsumer; - import org.apache.kafka.clients.consumer.ConsumerRecord; /** @@ -28,13 +26,8 @@ * * @author Marius Bogoevici * @author Gary Russell - * @author Sanghyeok An */ @FunctionalInterface public interface MessageListener extends GenericMessageListener> { - default void setCallbackForAsyncFailure( - BiConsumer, RuntimeException> asyncRetryCallback) { - // - } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java index a2774a8f4e..e208f50d1d 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java @@ -20,7 +20,6 @@ import java.time.Clock; import java.time.Instant; import java.util.Optional; -import java.util.function.BiConsumer; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -44,7 +43,6 @@ * @param the record key type. * @param the record value type. * @author Tomaz Fernandes - * @author Sanghyeok An * @since 2.7 * */ @@ -146,8 +144,4 @@ public void onMessage(ConsumerRecord data, Consumer consumer) { onMessage(data, null, consumer); } - @Override - public void setCallbackForAsyncFailure(BiConsumer, RuntimeException> callbackForAsyncFailureQueue) { - this.delegate.setCallbackForAsyncFailure(callbackForAsyncFailureQueue); - } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index 3e63bd0ec5..f08f087fa2 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -155,7 +155,8 @@ public abstract class MessagingMessageListenerAdapter implements ConsumerS private String correlationHeaderName = KafkaHeaders.CORRELATION_ID; - private BiConsumer, RuntimeException> callbackForAsyncFailureQueue; + @Nullable + private BiConsumer, RuntimeException> callbackForAsyncFailure; /** * Create an instance with the provided bean and method. @@ -678,9 +679,10 @@ protected void asyncFailure(Object request, @Nullable Acknowledgment acknowledgm catch (Throwable ex) { this.logger.error(t, () -> "Future, Mono, or suspend function was completed with an exception for " + source); acknowledge(acknowledgment); - if (canAsyncRetry(request, ex)) { + if (canAsyncRetry(request, ex) && + Objects.nonNull(this.callbackForAsyncFailure)) { ConsumerRecord record = (ConsumerRecord) request; - this.callbackForAsyncFailureQueue.accept(record, (RuntimeException) ex); + this.callbackForAsyncFailure.accept(record, (RuntimeException) ex); } } } @@ -884,8 +886,8 @@ private boolean rawByParameterIsType(Type parameterType, Type type) { return parameterType instanceof ParameterizedType pType && pType.getRawType().equals(type); } - protected void setAsyncFailureCallback(BiConsumer, RuntimeException> asyncRetryCallback) { - this.callbackForAsyncFailureQueue = asyncRetryCallback; + public void setCallbackForAsyncFailure(BiConsumer, RuntimeException> asyncRetryCallback) { + this.callbackForAsyncFailure = asyncRetryCallback; } /** diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java index ccfcb52b9a..6caa854e45 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/RecordMessagingMessageListenerAdapter.java @@ -17,7 +17,6 @@ package org.springframework.kafka.listener.adapter; import java.lang.reflect.Method; -import java.util.function.BiConsumer; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -86,9 +85,4 @@ public void onMessage(ConsumerRecord record, @Nullable Acknowledgment ackn invoke(record, acknowledgment, consumer, message); } - @Override - public void setCallbackForAsyncFailure( - BiConsumer, RuntimeException> asyncRetryCallback) { - setAsyncFailureCallback(asyncRetryCallback); - } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java index 0a6c64ecf5..64b66e3522 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 the original author or authors. + * Copyright 2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java index 5b949b18f9..27b5442557 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 the original author or authors. + * Copyright 2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -30,7 +29,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.Test; @@ -45,7 +43,6 @@ import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaAdmin; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.core.ProducerFactory; import org.springframework.kafka.listener.ContainerProperties; @@ -1210,13 +1207,6 @@ static class KafkaConsumerConfig { @Autowired EmbeddedKafkaBroker broker; - @Bean - KafkaAdmin kafkaAdmin() { - Map configs = new HashMap<>(); - configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, this.broker.getBrokersAsString()); - return new KafkaAdmin(configs); - } - @Bean ConsumerFactory consumerFactory() { Map props = KafkaTestUtils.consumerProps( diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java index 4eca91f805..81e67987a3 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 the original author or authors. + * Copyright 2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java index 7426cdfde8..1de4c4af63 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 the original author or authors. + * Copyright 2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -30,7 +29,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.Test; @@ -45,7 +43,6 @@ import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaAdmin; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.core.ProducerFactory; import org.springframework.kafka.listener.ContainerProperties; @@ -1178,13 +1175,6 @@ static class KafkaConsumerConfig { @Autowired EmbeddedKafkaBroker broker; - @Bean - KafkaAdmin kafkaAdmin() { - Map configs = new HashMap<>(); - configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, this.broker.getBrokersAsString()); - return new KafkaAdmin(configs); - } - @Bean ConsumerFactory consumerFactory() { Map props = KafkaTestUtils.consumerProps( From 645d7ced0ebb2ab5eefaba10bb6dc1ad58bd3c5b Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sat, 12 Oct 2024 07:38:06 +0900 Subject: [PATCH 12/30] Revert Copyright period. --- .../org/springframework/kafka/listener/MessageListener.java | 2 +- .../adapter/KafkaBackoffAwareMessageListenerAdapter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java index 5b20488d65..ab1c5a97b9 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/MessageListener.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 the original author or authors. + * Copyright 2015-2019 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java index e208f50d1d..2ed473b8a6 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 the original author or authors. + * Copyright 2018-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 0c89c92e90b4e0c956fb2bf64a10d190a7d5f639 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sat, 12 Oct 2024 07:42:57 +0900 Subject: [PATCH 13/30] Revert KafkaBackkOffAwareMessageListenerAdapter. --- .../adapter/KafkaBackoffAwareMessageListenerAdapter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java index 2ed473b8a6..d4eb63e903 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/KafkaBackoffAwareMessageListenerAdapter.java @@ -143,5 +143,4 @@ public void onMessage(ConsumerRecord data, Acknowledgment acknowledgment) public void onMessage(ConsumerRecord data, Consumer consumer) { onMessage(data, null, consumer); } - } From 3f6b447c4abf526edfc9e86ca67eefaf99f48087 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sat, 12 Oct 2024 08:05:38 +0900 Subject: [PATCH 14/30] Depends on general type on KafkaMessageListenerContainer. --- .../KafkaMessageListenerContainer.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index 957f64c4c9..0b04e2a5b4 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -107,8 +107,8 @@ import org.springframework.kafka.listener.ContainerProperties.AckMode; import org.springframework.kafka.listener.ContainerProperties.AssignmentCommitOption; import org.springframework.kafka.listener.ContainerProperties.EOSMode; +import org.springframework.kafka.listener.adapter.AbstractDelegatingMessageListenerAdapter; import org.springframework.kafka.listener.adapter.AsyncRepliesAware; -import org.springframework.kafka.listener.adapter.KafkaBackoffAwareMessageListenerAdapter; import org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter; import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.KafkaHeaders; @@ -903,19 +903,12 @@ else if (listener instanceof MessageListener) { this.observationEnabled = this.containerProperties.isObservationEnabled(); if (!AopUtils.isAopProxy(this.genericListener) && - this.genericListener instanceof KafkaBackoffAwareMessageListenerAdapter) { - KafkaBackoffAwareMessageListenerAdapter genListener = - (KafkaBackoffAwareMessageListenerAdapter) this.genericListener; - if (genListener.getDelegate() instanceof RecordMessagingMessageListenerAdapter) { - - RecordMessagingMessageListenerAdapter recordAdapterListener = - (RecordMessagingMessageListenerAdapter) genListener.getDelegate(); - - BiConsumer, RuntimeException> callbackForAsyncFailure = - (cRecord, ex) -> this.failedRecords.addLast(new FailedRecordTuple<>(cRecord, ex)); - recordAdapterListener.setCallbackForAsyncFailure(callbackForAsyncFailure); + this.genericListener instanceof AbstractDelegatingMessageListenerAdapter) { + AbstractDelegatingMessageListenerAdapter> genListener = + (AbstractDelegatingMessageListenerAdapter>) this.genericListener; + if (genListener.getDelegate() instanceof RecordMessagingMessageListenerAdapter adapterListener) { + adapterListener.setCallbackForAsyncFailure(getCallbackForAsyncFailure()); } - } } else { @@ -3391,6 +3384,10 @@ private Collection> getHighestOffsetRecords(ConsumerRecords .values(); } + private BiConsumer, RuntimeException> getCallbackForAsyncFailure() { + return (cRecord, ex) -> this.failedRecords.addLast(new FailedRecordTuple<>(cRecord, ex)); + } + @Override public void seek(String topic, int partition, long offset) { this.seeks.add(new TopicPartitionOffset(topic, partition, offset)); From 01de80e4a8b68218804838cc905de77b94122e1b Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sat, 12 Oct 2024 08:10:40 +0900 Subject: [PATCH 15/30] Remove sleep(1) from async retry test. --- ...eRetryTopicClassLevelIntegrationTests.java | 66 ------------------- ...eRetryTopicClassLevelIntegrationTests.java | 66 ------------------- 2 files changed, 132 deletions(-) diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java index 64b66e3522..c60613fe89 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java @@ -336,12 +336,6 @@ static class FirstTopicListener { public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { container.countDownLatch1.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Woooops... in topic " + receivedTopic); }); } @@ -358,12 +352,6 @@ static class SecondTopicListener { public CompletableFuture listenAgain(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch2); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new IllegalStateException("Another woooops... " + receivedTopic); }); } @@ -391,12 +379,6 @@ static class ThirdTopicListener { public CompletableFuture listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch3); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new MyRetryException("Annotated woooops... " + receivedTopic); }); } @@ -420,12 +402,6 @@ static class FourthTopicListener { public CompletableFuture listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch4); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new IllegalStateException("Another woooops... " + receivedTopic); }); } @@ -472,12 +448,6 @@ public CompletableFuture listenWithAnnotation(String message, @Header(Kafk this.topics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Annotated woooops... " + receivedTopic); }); } @@ -505,12 +475,6 @@ public CompletableFuture listenWithAnnotation2(String message, @Header(Kaf this.topics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { container.countDownLatch52.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Annotated woooops... " + receivedTopic); }); } @@ -537,12 +501,6 @@ public CompletableFuture listenNoDlt( @SuppressWarnings("unused") Acknowledgment ack) { return CompletableFuture.supplyAsync(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch6); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new IllegalStateException("Another woooops... " + receivedTopic); }); } @@ -566,12 +524,6 @@ static class NoRetryTopicListener { public CompletableFuture listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatchNoRetry); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); }); } @@ -601,12 +553,6 @@ public CompletableFuture listen1(String message, @Header(KafkaHeaders.RECE this.topics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { container.countDownLatchReuseOne.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Another woooops... " + receivedTopic); }); } @@ -632,12 +578,6 @@ public CompletableFuture listen2(String message, @Header(KafkaHeaders.RECE this.topics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { container.countDownLatchReuseTwo.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Another woooops... " + receivedTopic); }); } @@ -659,12 +599,6 @@ public CompletableFuture listen3(String message, @Header(KafkaHeaders.RECE this.topics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { container.countDownLatchReuseThree.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Another woooops... " + receivedTopic); }); diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java index 81e67987a3..6db3e938f4 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java @@ -336,12 +336,6 @@ public Mono listen( @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { container.countDownLatch1.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Woooops... in topic " + receivedTopic); }).then(); } @@ -360,12 +354,6 @@ public Mono listenAgain( @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch2); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new IllegalStateException("Another woooops... " + receivedTopic); }).then(); } @@ -395,12 +383,6 @@ public Mono listenWithAnnotation( @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch3); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new MyRetryException("Annotated woooops... " + receivedTopic); }).then(); } @@ -426,12 +408,6 @@ public Mono listenNoDlt( @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch4); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new IllegalStateException("Another woooops... " + receivedTopic); }).then(); } @@ -478,12 +454,6 @@ public Mono listenWithAnnotation(String message, @Header(KafkaHeaders.RECE this.topics.add(receivedTopic); return Mono.fromCallable(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Annotated woooops... " + receivedTopic); }).then(); } @@ -511,12 +481,6 @@ public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.REC this.topics.add(receivedTopic); return Mono.fromCallable(() -> { container.countDownLatch52.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Annotated woooops... " + receivedTopic); }).then(); } @@ -543,12 +507,6 @@ public Mono listenNoDlt( @SuppressWarnings("unused") Acknowledgment ack) { return Mono.fromCallable(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatch6); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new IllegalStateException("Another woooops... " + receivedTopic); }).then(); } @@ -572,12 +530,6 @@ static class NoRetryTopicListener { public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { container.countDownIfNotKnown(receivedTopic, container.countDownLatchNoRetry); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); }).then(); } @@ -607,12 +559,6 @@ public Mono listen1(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) S this.topics.add(receivedTopic); return Mono.fromCallable(() -> { container.countDownLatchReuseOne.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Another woooops... " + receivedTopic); }).then(); } @@ -638,12 +584,6 @@ public Mono listen2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) S this.topics.add(receivedTopic); return Mono.fromCallable(() -> { container.countDownLatchReuseTwo.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Another woooops... " + receivedTopic); }).then(); } @@ -665,12 +605,6 @@ public Mono listen3(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) S this.topics.add(receivedTopic); return Mono.fromCallable(() -> { container.countDownLatchReuseThree.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Another woooops... " + receivedTopic); }).then(); } From 97f05bebeb2ad6060fd50d1383614eead33e8086 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sat, 12 Oct 2024 09:23:23 +0900 Subject: [PATCH 16/30] Remove waitAWhile(). --- ...pletableFutureRetryTopicScenarioTests.java | 136 ++------- .../AsyncMonoRetryTopicScenarioTests.java | 267 +++++++----------- 2 files changed, 133 insertions(+), 270 deletions(-) diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java index 27b5442557..f29b2ccb5c 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -161,16 +161,12 @@ void allFailCaseTest( assertThat(awaitLatch(latchContainer.countDownLatch0)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch0)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(TEST_TOPIC0 + "-retry"); assertThat(zeroTopicListener.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); assertThat(zeroTopicListener.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - assertThat(latchContainer.extraCountDownLatch0.getCount()).isEqualTo(1000 - CountDownLatchContainer.COUNT0); assertThat(myCustomDltProcessor0.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch0.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT0); } @Test @@ -215,15 +211,11 @@ void firstShortFailAndLastLongSuccessRetryTest( assertThat(awaitLatch(latchContainer.countDownLatch1)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch1)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(latchContainer.extraCountDownLatch1.getCount()).isEqualTo(1000 - CountDownLatchContainer.COUNT1); assertThat(testTopicListener1.receivedMsgs).containsExactly(expectedReceivedMsgs); assertThat(testTopicListener1.receivedTopics).containsExactly(expectedReceivedTopics); assertThat(myCustomDltProcessor1.receivedMsg).containsExactly(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch1.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT1); } @Test @@ -268,14 +260,11 @@ void firstLongSuccessAndLastShortFailed( assertThat(awaitLatch(latchContainer.countDownLatch2)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch2)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); assertThat(zero2TopicListener.receivedMsgs).containsExactly(expectedReceivedMsgs); assertThat(zero2TopicListener.receivedTopics).containsExactly(expectedReceivedTopics); assertThat(myCustomDltProcessor2.receivedMsg).containsExactly(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch2.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT2); } @Test @@ -335,14 +324,11 @@ void longFailMsgTwiceThenShortSucessMsgThird( assertThat(awaitLatch(latchContainer.countDownLatch3)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch3)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); assertThat(testTopicListener3.receivedMsgs).containsExactly(expectedReceivedMsgs); assertThat(testTopicListener3.receivedTopics).containsExactly(expectedReceivedTopics); assertThat(myCustomDltProcessor3.receivedMsg).containsExactly(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch3.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT3); } @Test @@ -399,14 +385,11 @@ void longSuccessMsgTwiceThenShortFailMsgTwice( assertThat(awaitLatch(latchContainer.countDownLatch4)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch4)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); assertThat(topicListener4.receivedMsgs).containsExactly(expectedReceivedMsgs); assertThat(topicListener4.receivedTopics).containsExactly(expectedReceivedTopics); assertThat(myCustomDltProcessor4.receivedMsg).containsExactly(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch4.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT4); } @Test @@ -453,14 +436,11 @@ void oneLongSuccessMsgBetweenHunderedShortFailMsg( assertThat(awaitLatch(latchContainer.countDownLatch5)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch5)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); assertThat(topicListener5.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); assertThat(topicListener5.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); assertThat(myCustomDltProcessor5.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch5.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT5); } @Test @@ -506,8 +486,6 @@ void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime( assertThat(awaitLatch(latchContainer.countDownLatch6)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch6)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); long actualReceivedSuccessMsgCount = topicListener6.receivedMsgs.stream() @@ -536,7 +514,6 @@ void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime( assertThat(actualReceivedRetryTopicMsgCount).isEqualTo(expectedReceivedRetryTopicCount); assertThat(myCustomDltProcessor6.receivedMsg.size()).isEqualTo(expectedReceivedDltMsgCount); - assertThat(latchContainer.extraDltCountdownLatch6.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT6); } private boolean awaitLatch(CountDownLatch latch) { @@ -550,23 +527,6 @@ private boolean awaitLatch(CountDownLatch latch) { } - private void waitAWhile(ThreadPoolTaskExecutor executor, int sleep) { - CountDownLatch countDownLatch = new CountDownLatch(1); - - Thread thread = new Thread(() -> { - try { - Thread.sleep(sleep); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - countDownLatch.countDown(); - }); - - executor.execute(thread); - assertThat(awaitLatch(countDownLatch)).isTrue(); - } - @KafkaListener( id = "0-topicId", topics = TEST_TOPIC0, @@ -588,14 +548,7 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEI this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch0.countDown(); container.countDownLatch0.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Woooops... in topic " + receivedTopic); }); } @@ -623,14 +576,15 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch1.countDown(); - container.countDownLatch1.countDown(); try { Thread.sleep(Integer.parseInt(message)); } catch (InterruptedException e) { throw new RuntimeException(e); } + finally { + container.countDownLatch1.countDown(); + } if (message.equals("1")) { throw new RuntimeException("Woooops... in topic " + receivedTopic); } @@ -662,15 +616,15 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC this.receivedTopics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch2.countDown(); - container.countDownLatch2.countDown(); - long count = container.countDownLatch2.getCount(); try { Thread.sleep(Integer.parseInt(message)); } catch (InterruptedException e) { throw new RuntimeException(e); } + finally { + container.countDownLatch2.countDown(); + } if (message.equals("1")) { throw new RuntimeException("Woooops... in topic " + receivedTopic); @@ -706,15 +660,15 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC this.receivedTopics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch3.countDown(); - container.countDownLatch3.countDown(); - long count = container.countDownLatch3.getCount(); try { Thread.sleep(Integer.parseInt(message)); } catch (InterruptedException e) { throw new RuntimeException(e); } + finally { + container.countDownLatch3.countDown(); + } if (message.equals(LONG_FAIL_MSG)) { throw new RuntimeException("Woooops... in topic " + receivedTopic); @@ -750,14 +704,15 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC this.receivedTopics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch4.countDown(); - container.countDownLatch4.countDown(); try { Thread.sleep(Integer.parseInt(message)); } catch (InterruptedException e) { throw new RuntimeException(e); } + finally { + container.countDownLatch4.countDown(); + } if (message.equals(SHORT_FAIL_MSG)) { throw new RuntimeException("Woooops... in topic " + receivedTopic); @@ -793,14 +748,15 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC this.receivedTopics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch5.countDown(); - container.countDownLatch5.countDown(); try { Thread.sleep(Integer.parseInt(message)); } catch (InterruptedException e) { throw new RuntimeException(e); } + finally { + container.countDownLatch5.countDown(); + } if (message.startsWith(SHORT_FAIL_MSG)) { throw new RuntimeException("Woooops... in topic " + receivedTopic); @@ -836,9 +792,6 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC this.receivedTopics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.extraCountDownLatch6.countDown(); - container.countDownLatch6.countDown(); - String[] split = message.split(","); String sleepAWhile = split[0]; String failOrSuccess = split[1]; @@ -849,6 +802,9 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC catch (InterruptedException e) { throw new RuntimeException(e); } + finally { + container.countDownLatch6.countDown(); + } if (failOrSuccess.equals("f")) { throw new RuntimeException("Woooops... in topic " + receivedTopic); @@ -866,84 +822,56 @@ static class CountDownLatchContainer { CountDownLatch countDownLatch0 = new CountDownLatch(COUNT0); - CountDownLatch extraCountDownLatch0 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch0 = new CountDownLatch(DLT_COUNT0); - CountDownLatch extraDltCountdownLatch0 = new CountDownLatch(1000); - static int COUNT1 = 6; static int DLT_COUNT1 = 1; CountDownLatch countDownLatch1 = new CountDownLatch(COUNT1); - CountDownLatch extraCountDownLatch1 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch1 = new CountDownLatch(DLT_COUNT1); - CountDownLatch extraDltCountdownLatch1 = new CountDownLatch(1000); - static int COUNT2 = 6; static int DLT_COUNT2 = 1; CountDownLatch countDownLatch2 = new CountDownLatch(COUNT2); - CountDownLatch extraCountDownLatch2 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch2 = new CountDownLatch(DLT_COUNT2); - CountDownLatch extraDltCountdownLatch2 = new CountDownLatch(1000); - static int COUNT3 = 13; static int DLT_COUNT3 = 2; CountDownLatch countDownLatch3 = new CountDownLatch(COUNT3); - CountDownLatch extraCountDownLatch3 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch3 = new CountDownLatch(DLT_COUNT3); - CountDownLatch extraDltCountdownLatch3 = new CountDownLatch(1000); - static int COUNT4 = 12; static int DLT_COUNT4 = 2; CountDownLatch countDownLatch4 = new CountDownLatch(COUNT4); - CountDownLatch extraCountDownLatch4 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch4 = new CountDownLatch(DLT_COUNT4); - CountDownLatch extraDltCountdownLatch4 = new CountDownLatch(1000); - static int COUNT5 = 501; static int DLT_COUNT5 = 100; CountDownLatch countDownLatch5 = new CountDownLatch(COUNT5); - CountDownLatch extraCountDownLatch5 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch5 = new CountDownLatch(DLT_COUNT5); - CountDownLatch extraDltCountdownLatch5 = new CountDownLatch(1000); - static int COUNT6 = 250; static int DLT_COUNT6 = 50; CountDownLatch countDownLatch6 = new CountDownLatch(COUNT6); - CountDownLatch extraCountDownLatch6 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch6 = new CountDownLatch(DLT_COUNT6); - CountDownLatch extraDltCountdownLatch6 = new CountDownLatch(1000); - CountDownLatch customErrorHandlerCountdownLatch = new CountDownLatch(6); CountDownLatch customMessageConverterCountdownLatch = new CountDownLatch(6); @@ -955,23 +883,18 @@ static class MyCustomDltProcessor { final List receivedMsg = new ArrayList<>(); MyCustomDltProcessor(KafkaTemplate kafkaTemplate, - CountDownLatch latch, - CountDownLatch extraLatch) { + CountDownLatch latch) { this.kafkaTemplate = kafkaTemplate; this.latch = latch; - this.extraLatch = extraLatch; } private final KafkaTemplate kafkaTemplate; private final CountDownLatch latch; - private final CountDownLatch extraLatch; - public void processDltMessage(String message) { this.receivedMsg.add(message); latch.countDown(); - extraLatch.countDown(); } } @@ -1119,8 +1042,7 @@ MyCustomDltProcessor myCustomDltProcessor0( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch0, - latchContainer.extraDltCountdownLatch0); + latchContainer.dltCountdownLatch0); } @Bean @@ -1128,8 +1050,7 @@ MyCustomDltProcessor myCustomDltProcessor1( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch1, - latchContainer.extraDltCountdownLatch1); + latchContainer.dltCountdownLatch1); } @Bean @@ -1137,8 +1058,7 @@ MyCustomDltProcessor myCustomDltProcessor2( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch2, - latchContainer.extraDltCountdownLatch2); + latchContainer.dltCountdownLatch2); } @Bean @@ -1146,8 +1066,7 @@ MyCustomDltProcessor myCustomDltProcessor3( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch3, - latchContainer.extraDltCountdownLatch3); + latchContainer.dltCountdownLatch3); } @Bean @@ -1155,8 +1074,7 @@ MyCustomDltProcessor myCustomDltProcessor4( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch4, - latchContainer.extraDltCountdownLatch4); + latchContainer.dltCountdownLatch4); } @Bean @@ -1164,8 +1082,7 @@ MyCustomDltProcessor myCustomDltProcessor5( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch5, - latchContainer.extraDltCountdownLatch5); + latchContainer.dltCountdownLatch5); } @Bean @@ -1173,8 +1090,7 @@ MyCustomDltProcessor myCustomDltProcessor6( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch6, - latchContainer.extraDltCountdownLatch6); + latchContainer.dltCountdownLatch6); } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java index 1de4c4af63..cf6bfe66e4 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -19,7 +19,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; -import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -163,16 +162,12 @@ void allFailCaseTest( assertThat(awaitLatch(latchContainer.countDownLatch0)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch0)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(TEST_TOPIC0 + "-retry"); assertThat(zeroTopicListener.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); assertThat(zeroTopicListener.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - assertThat(latchContainer.extraCountDownLatch0.getCount()).isEqualTo(1000 - CountDownLatchContainer.COUNT0); assertThat(myCustomDltProcessor0.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch0.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT0); } @Test @@ -216,15 +211,11 @@ void firstShortFailAndLastLongSuccessRetryTest( assertThat(awaitLatch(latchContainer.countDownLatch1)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch1)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(latchContainer.extraCountDownLatch1.getCount()).isEqualTo(1000 - CountDownLatchContainer.COUNT1); assertThat(testTopicListener1.receivedMsgs).containsExactly(expectedReceivedMsgs); assertThat(testTopicListener1.receivedTopics).containsExactly(expectedReceivedTopics); assertThat(myCustomDltProcessor1.receivedMsg).containsExactly(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch1.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT1); } @Test @@ -268,14 +259,11 @@ void firstLongSuccessAndLastShortFailed( assertThat(awaitLatch(latchContainer.countDownLatch2)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch2)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); assertThat(zero2TopicListener.receivedMsgs).containsExactly(expectedReceivedMsgs); assertThat(zero2TopicListener.receivedTopics).containsExactly(expectedReceivedTopics); assertThat(myCustomDltProcessor2.receivedMsg).containsExactly(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch2.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT2); } @Test @@ -335,14 +323,11 @@ void longFailMsgTwiceThenShortSuccessMsgThird( assertThat(awaitLatch(latchContainer.countDownLatch3)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch3)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); assertThat(testTopicListener3.receivedMsgs).containsExactly(expectedReceivedMsgs); assertThat(testTopicListener3.receivedTopics).containsExactly(expectedReceivedTopics); assertThat(myCustomDltProcessor3.receivedMsg).containsExactly(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch3.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT3); } @Test @@ -399,14 +384,11 @@ void longSuccessMsgTwiceThenShortFailMsgTwice( assertThat(awaitLatch(latchContainer.countDownLatch4)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch4)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); assertThat(topicListener4.receivedMsgs).containsExactly(expectedReceivedMsgs); assertThat(topicListener4.receivedTopics).containsExactly(expectedReceivedTopics); assertThat(myCustomDltProcessor4.receivedMsg).containsExactly(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch4.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT4); } @Test @@ -452,14 +434,11 @@ void oneLongSuccessMsgBetweenHunderedShortFailMsg( assertThat(awaitLatch(latchContainer.countDownLatch5)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch5)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); assertThat(topicListener5.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); assertThat(topicListener5.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); assertThat(myCustomDltProcessor5.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); - assertThat(latchContainer.extraDltCountdownLatch5.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT5); } @Test @@ -505,8 +484,6 @@ void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime( assertThat(awaitLatch(latchContainer.countDownLatch6)).isTrue(); assertThat(awaitLatch(latchContainer.dltCountdownLatch6)).isTrue(); - waitAWhile(executor, 10000); - assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); long actualReceivedSuccessMsgCount = topicListener6.receivedMsgs.stream() @@ -535,7 +512,6 @@ void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime( assertThat(actualReceivedRetryTopicMsgCount).isEqualTo(expectedReceivedRetryTopicCount); assertThat(myCustomDltProcessor6.receivedMsg.size()).isEqualTo(expectedReceivedDltMsgCount); - assertThat(latchContainer.extraDltCountdownLatch6.getCount()).isEqualTo(1000 - CountDownLatchContainer.DLT_COUNT6); } private boolean awaitLatch(CountDownLatch latch) { @@ -548,24 +524,6 @@ private boolean awaitLatch(CountDownLatch latch) { } } - private void waitAWhile(ThreadPoolTaskExecutor executor, int sleep) { - CountDownLatch countDownLatch = new CountDownLatch(1); - - Thread thread = new Thread(() -> { - try { - Thread.sleep(sleep); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - countDownLatch.countDown(); - } - ); - - executor.execute(thread); - assertThat(awaitLatch(countDownLatch)).isTrue(); - } - @KafkaListener( id = "0-topicId", topics = TEST_TOPIC0, @@ -587,14 +545,7 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) St this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); return Mono.fromCallable(() -> { - container.extraCountDownLatch0.countDown(); container.countDownLatch0.countDown(); - try { - Thread.sleep(1); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } throw new RuntimeException("Woooops... in topic " + receivedTopic); }).then(); } @@ -622,16 +573,22 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); - container.extraCountDownLatch1.countDown(); - container.countDownLatch1.countDown(); - return Mono - .delay(Duration.ofMillis(Integer.parseInt(message))) - .flatMap(delay -> { - if ("1".equals(message)) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); - }); + return Mono.fromCallable(() -> { + try { + Thread.sleep(Integer.parseInt(message)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + finally { + container.countDownLatch1.countDown(); + } + if (message.equals("1")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } } @@ -656,16 +613,22 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); - container.extraCountDownLatch2.countDown(); - container.countDownLatch2.countDown(); - return Mono - .delay(Duration.ofMillis(Integer.parseInt(message))) - .flatMap(delay -> { - if ("1".equals(message)) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); - }); + return Mono.fromCallable(() -> { + try { + Thread.sleep(Integer.parseInt(message)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + finally { + container.countDownLatch2.countDown(); + } + + if (message.equals("1")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); } } @@ -694,16 +657,24 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); - container.extraCountDownLatch3.countDown(); - container.countDownLatch3.countDown(); - return Mono - .delay(Duration.ofMillis(Integer.parseInt(message))) - .flatMap(delay -> { - if (message.equals(LONG_FAIL_MSG)) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); - }); + return Mono.fromCallable(() -> { + try { + Thread.sleep(Integer.parseInt(message)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + finally { + container.countDownLatch3.countDown(); + } + + if (message.equals(LONG_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + + } } @@ -732,16 +703,23 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); - container.extraCountDownLatch4.countDown(); - container.countDownLatch4.countDown(); - return Mono - .delay(Duration.ofMillis(Integer.parseInt(message))) - .flatMap(delay -> { - if (message.equals(SHORT_FAIL_MSG)) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); - }); + return Mono.fromCallable(() -> { + try { + Thread.sleep(Integer.parseInt(message)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + finally { + container.countDownLatch4.countDown(); + } + + if (message.equals(SHORT_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + return "Task Completed"; + }); + } } @@ -770,17 +748,22 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); - container.extraCountDownLatch5.countDown(); - container.countDownLatch5.countDown(); - return Mono - .delay(Duration.ofMillis(Integer.parseInt(message))) - .flatMap(delay -> { - if (message.startsWith(SHORT_FAIL_MSG)) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); + return Mono.fromCallable(() -> { + try { + Thread.sleep(Integer.parseInt(message)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + finally { + container.countDownLatch5.countDown(); + } + + if (message.startsWith(SHORT_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); } - ); + return "Task Completed"; + }); } } @@ -809,22 +792,26 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); - container.extraCountDownLatch6.countDown(); - container.countDownLatch6.countDown(); + return Mono.fromCallable(() -> { + String[] split = message.split(","); + String sleepAWhile = split[0]; + String failOrSuccess = split[1]; - String[] split = message.split(","); - String sleepAWhile = split[0]; - String failOrSuccess = split[1]; + try { + Thread.sleep(Integer.parseInt(sleepAWhile)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + finally { + container.countDownLatch6.countDown(); + } - return Mono - .delay(Duration.ofMillis(Integer.parseInt(sleepAWhile))) - .flatMap(delay -> { - if (failOrSuccess.equals("f")) { - return Mono.error(new RuntimeException("Woooops... in topic " + receivedTopic)); - } - return Mono.just("Task Completed"); + if (failOrSuccess.equals("f")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); } - ); + return "Task Completed"; + }); } } @@ -836,84 +823,56 @@ static class CountDownLatchContainer { CountDownLatch countDownLatch0 = new CountDownLatch(COUNT0); - CountDownLatch extraCountDownLatch0 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch0 = new CountDownLatch(DLT_COUNT0); - CountDownLatch extraDltCountdownLatch0 = new CountDownLatch(1000); - static int COUNT1 = 6; static int DLT_COUNT1 = 1; CountDownLatch countDownLatch1 = new CountDownLatch(COUNT1); - CountDownLatch extraCountDownLatch1 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch1 = new CountDownLatch(DLT_COUNT1); - CountDownLatch extraDltCountdownLatch1 = new CountDownLatch(1000); - static int COUNT2 = 6; static int DLT_COUNT2 = 1; CountDownLatch countDownLatch2 = new CountDownLatch(COUNT2); - CountDownLatch extraCountDownLatch2 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch2 = new CountDownLatch(DLT_COUNT2); - CountDownLatch extraDltCountdownLatch2 = new CountDownLatch(1000); - static int COUNT3 = 13; static int DLT_COUNT3 = 2; CountDownLatch countDownLatch3 = new CountDownLatch(COUNT3); - CountDownLatch extraCountDownLatch3 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch3 = new CountDownLatch(DLT_COUNT3); - CountDownLatch extraDltCountdownLatch3 = new CountDownLatch(1000); - static int COUNT4 = 12; static int DLT_COUNT4 = 2; CountDownLatch countDownLatch4 = new CountDownLatch(COUNT4); - CountDownLatch extraCountDownLatch4 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch4 = new CountDownLatch(DLT_COUNT4); - CountDownLatch extraDltCountdownLatch4 = new CountDownLatch(1000); - static int COUNT5 = 501; static int DLT_COUNT5 = 100; CountDownLatch countDownLatch5 = new CountDownLatch(COUNT5); - CountDownLatch extraCountDownLatch5 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch5 = new CountDownLatch(DLT_COUNT5); - CountDownLatch extraDltCountdownLatch5 = new CountDownLatch(1000); - static int COUNT6 = 250; static int DLT_COUNT6 = 50; CountDownLatch countDownLatch6 = new CountDownLatch(COUNT6); - CountDownLatch extraCountDownLatch6 = new CountDownLatch(1000); - CountDownLatch dltCountdownLatch6 = new CountDownLatch(DLT_COUNT6); - CountDownLatch extraDltCountdownLatch6 = new CountDownLatch(1000); - CountDownLatch customErrorHandlerCountdownLatch = new CountDownLatch(6); CountDownLatch customMessageConverterCountdownLatch = new CountDownLatch(6); @@ -925,23 +884,18 @@ static class MyCustomDltProcessor { final List receivedMsg = new ArrayList<>(); MyCustomDltProcessor(KafkaTemplate kafkaTemplate, - CountDownLatch latch, - CountDownLatch extraLatch) { + CountDownLatch latch) { this.kafkaTemplate = kafkaTemplate; this.latch = latch; - this.extraLatch = extraLatch; } private final KafkaTemplate kafkaTemplate; private final CountDownLatch latch; - private final CountDownLatch extraLatch; - public void processDltMessage(String message) { this.receivedMsg.add(message); latch.countDown(); - extraLatch.countDown(); } } @@ -1088,8 +1042,7 @@ MyCustomDltProcessor myCustomDltProcessor0( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch0, - latchContainer.extraDltCountdownLatch0); + latchContainer.dltCountdownLatch0); } @Bean @@ -1097,8 +1050,7 @@ MyCustomDltProcessor myCustomDltProcessor1( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch1, - latchContainer.extraDltCountdownLatch1); + latchContainer.dltCountdownLatch1); } @Bean @@ -1106,8 +1058,7 @@ MyCustomDltProcessor myCustomDltProcessor2( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch2, - latchContainer.extraDltCountdownLatch2); + latchContainer.dltCountdownLatch2); } @Bean @@ -1115,8 +1066,7 @@ MyCustomDltProcessor myCustomDltProcessor3( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch3, - latchContainer.extraDltCountdownLatch3); + latchContainer.dltCountdownLatch3); } @Bean @@ -1124,8 +1074,7 @@ MyCustomDltProcessor myCustomDltProcessor4( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch4, - latchContainer.extraDltCountdownLatch4); + latchContainer.dltCountdownLatch4); } @Bean @@ -1133,8 +1082,7 @@ MyCustomDltProcessor myCustomDltProcessor5( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch5, - latchContainer.extraDltCountdownLatch5); + latchContainer.dltCountdownLatch5); } @Bean @@ -1142,8 +1090,7 @@ MyCustomDltProcessor myCustomDltProcessor6( KafkaTemplate kafkaTemplate, CountDownLatchContainer latchContainer) { return new MyCustomDltProcessor(kafkaTemplate, - latchContainer.dltCountdownLatch6, - latchContainer.extraDltCountdownLatch6); + latchContainer.dltCountdownLatch6); } } From d219037de7d31ef76b91e20c353ae8ff215035c0 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sat, 12 Oct 2024 09:56:14 +0900 Subject: [PATCH 17/30] Add java docs. --- .../listener/adapter/MessagingMessageListenerAdapter.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index f08f087fa2..957abdd752 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -886,6 +886,13 @@ private boolean rawByParameterIsType(Type parameterType, Type type) { return parameterType instanceof ParameterizedType pType && pType.getRawType().equals(type); } + /** + * Sets the retry callback for failures of both {@link CompletableFuture} and {@link Mono}. + * {@link MessagingMessageListenerAdapter#asyncFailure(Object, Acknowledgment, Consumer, Throwable, Message)} + * will invoke {@link MessagingMessageListenerAdapter#callbackForAsyncFailure} when + * {@link CompletableFuture} or {@link Mono} fails to complete. + * @param asyncRetryCallback the callback for async retry. + */ public void setCallbackForAsyncFailure(BiConsumer, RuntimeException> asyncRetryCallback) { this.callbackForAsyncFailure = asyncRetryCallback; } From e8378da9a0f20ac29d304d8ef5828c109e43147d Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sat, 12 Oct 2024 09:57:34 +0900 Subject: [PATCH 18/30] Add author --- .../kafka/listener/KafkaMessageListenerContainer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index 0b04e2a5b4..283755035a 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -171,6 +171,7 @@ * @author Mikael Carlstedt * @author Borahm Lee * @author Lokesh Alamuri + * @author Sanghyeok An */ public class KafkaMessageListenerContainer // NOSONAR line count extends AbstractMessageListenerContainer implements ConsumerPauseResumeEventPublisher { From 967a7c4a5c7a44c846da32b64d37dc4acdc3e999 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sat, 12 Oct 2024 10:26:17 +0900 Subject: [PATCH 19/30] Fixes flaky test. --- ...pletableFutureRetryTopicScenarioTests.java | 24 +++++++++---------- .../AsyncMonoRetryTopicScenarioTests.java | 24 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java index f29b2ccb5c..1dd852e121 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -212,10 +212,10 @@ void firstShortFailAndLastLongSuccessRetryTest( assertThat(awaitLatch(latchContainer.dltCountdownLatch1)).isTrue(); assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(testTopicListener1.receivedMsgs).containsExactly(expectedReceivedMsgs); - assertThat(testTopicListener1.receivedTopics).containsExactly(expectedReceivedTopics); + assertThat(testTopicListener1.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(testTopicListener1.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - assertThat(myCustomDltProcessor1.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(myCustomDltProcessor1.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); } @Test @@ -261,10 +261,10 @@ void firstLongSuccessAndLastShortFailed( assertThat(awaitLatch(latchContainer.dltCountdownLatch2)).isTrue(); assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(zero2TopicListener.receivedMsgs).containsExactly(expectedReceivedMsgs); - assertThat(zero2TopicListener.receivedTopics).containsExactly(expectedReceivedTopics); + assertThat(zero2TopicListener.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(zero2TopicListener.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - assertThat(myCustomDltProcessor2.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(myCustomDltProcessor2.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); } @Test @@ -325,10 +325,10 @@ void longFailMsgTwiceThenShortSucessMsgThird( assertThat(awaitLatch(latchContainer.dltCountdownLatch3)).isTrue(); assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(testTopicListener3.receivedMsgs).containsExactly(expectedReceivedMsgs); - assertThat(testTopicListener3.receivedTopics).containsExactly(expectedReceivedTopics); + assertThat(testTopicListener3.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(testTopicListener3.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - assertThat(myCustomDltProcessor3.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(myCustomDltProcessor3.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); } @Test @@ -386,10 +386,10 @@ void longSuccessMsgTwiceThenShortFailMsgTwice( assertThat(awaitLatch(latchContainer.dltCountdownLatch4)).isTrue(); assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(topicListener4.receivedMsgs).containsExactly(expectedReceivedMsgs); - assertThat(topicListener4.receivedTopics).containsExactly(expectedReceivedTopics); + assertThat(topicListener4.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(topicListener4.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - assertThat(myCustomDltProcessor4.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(myCustomDltProcessor4.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); } @Test diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java index cf6bfe66e4..2392abc6d4 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -212,10 +212,10 @@ void firstShortFailAndLastLongSuccessRetryTest( assertThat(awaitLatch(latchContainer.dltCountdownLatch1)).isTrue(); assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(testTopicListener1.receivedMsgs).containsExactly(expectedReceivedMsgs); - assertThat(testTopicListener1.receivedTopics).containsExactly(expectedReceivedTopics); + assertThat(testTopicListener1.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(testTopicListener1.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - assertThat(myCustomDltProcessor1.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(myCustomDltProcessor1.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); } @Test @@ -260,10 +260,10 @@ void firstLongSuccessAndLastShortFailed( assertThat(awaitLatch(latchContainer.dltCountdownLatch2)).isTrue(); assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(zero2TopicListener.receivedMsgs).containsExactly(expectedReceivedMsgs); - assertThat(zero2TopicListener.receivedTopics).containsExactly(expectedReceivedTopics); + assertThat(zero2TopicListener.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(zero2TopicListener.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - assertThat(myCustomDltProcessor2.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(myCustomDltProcessor2.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); } @Test @@ -324,10 +324,10 @@ void longFailMsgTwiceThenShortSuccessMsgThird( assertThat(awaitLatch(latchContainer.dltCountdownLatch3)).isTrue(); assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(testTopicListener3.receivedMsgs).containsExactly(expectedReceivedMsgs); - assertThat(testTopicListener3.receivedTopics).containsExactly(expectedReceivedTopics); + assertThat(testTopicListener3.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(testTopicListener3.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - assertThat(myCustomDltProcessor3.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(myCustomDltProcessor3.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); } @Test @@ -385,10 +385,10 @@ void longSuccessMsgTwiceThenShortFailMsgTwice( assertThat(awaitLatch(latchContainer.dltCountdownLatch4)).isTrue(); assertThat(destinationTopic.getDestinationName()).isEqualTo(expectedRetryTopic); - assertThat(topicListener4.receivedMsgs).containsExactly(expectedReceivedMsgs); - assertThat(topicListener4.receivedTopics).containsExactly(expectedReceivedTopics); + assertThat(topicListener4.receivedMsgs).containsExactlyInAnyOrder(expectedReceivedMsgs); + assertThat(topicListener4.receivedTopics).containsExactlyInAnyOrder(expectedReceivedTopics); - assertThat(myCustomDltProcessor4.receivedMsg).containsExactly(expectedDltMsgs); + assertThat(myCustomDltProcessor4.receivedMsg).containsExactlyInAnyOrder(expectedDltMsgs); } @Test From f2c67c7d5f45068870be582f1ac27a1aa7631dad Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 15 Oct 2024 08:12:22 +0900 Subject: [PATCH 20/30] Modify contract of callback for async failure. --- .../kafka/listener/KafkaMessageListenerContainer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index 283755035a..66f623fc4d 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -908,7 +908,7 @@ else if (listener instanceof MessageListener) { AbstractDelegatingMessageListenerAdapter> genListener = (AbstractDelegatingMessageListenerAdapter>) this.genericListener; if (genListener.getDelegate() instanceof RecordMessagingMessageListenerAdapter adapterListener) { - adapterListener.setCallbackForAsyncFailure(getCallbackForAsyncFailure()); + adapterListener.setCallbackForAsyncFailure(this::callbackForAsyncFailure); } } } @@ -3385,8 +3385,8 @@ private Collection> getHighestOffsetRecords(ConsumerRecords .values(); } - private BiConsumer, RuntimeException> getCallbackForAsyncFailure() { - return (cRecord, ex) -> this.failedRecords.addLast(new FailedRecordTuple<>(cRecord, ex)); + private void callbackForAsyncFailure(ConsumerRecord cRecord, RuntimeException ex) { + this.failedRecords.addLast(new FailedRecordTuple<>(cRecord, ex)); } @Override From 56e6fd3c140336272d866354317d8693bf636973 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 15 Oct 2024 08:13:58 +0900 Subject: [PATCH 21/30] Fixes java docs for setCallbackFroAsyncFailure. --- .../listener/adapter/MessagingMessageListenerAdapter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index 957abdd752..07fbac0030 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -887,11 +887,12 @@ private boolean rawByParameterIsType(Type parameterType, Type type) { } /** - * Sets the retry callback for failures of both {@link CompletableFuture} and {@link Mono}. + * Set the retry callback for failures of both {@link CompletableFuture} and {@link Mono}. * {@link MessagingMessageListenerAdapter#asyncFailure(Object, Acknowledgment, Consumer, Throwable, Message)} * will invoke {@link MessagingMessageListenerAdapter#callbackForAsyncFailure} when * {@link CompletableFuture} or {@link Mono} fails to complete. * @param asyncRetryCallback the callback for async retry. + * @since 3.3 */ public void setCallbackForAsyncFailure(BiConsumer, RuntimeException> asyncRetryCallback) { this.callbackForAsyncFailure = asyncRetryCallback; From 5e70c76e3c15badefb32448db0e9ee70c325b5b7 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 15 Oct 2024 09:42:44 +0900 Subject: [PATCH 22/30] Fix the tests for async failure retry. --- ...eRetryTopicClassLevelIntegrationTests.java | 121 ++++++++++++--- ...pletableFutureRetryTopicScenarioTests.java | 139 ++++++++---------- ...eRetryTopicClassLevelIntegrationTests.java | 122 ++++++++++++--- .../AsyncMonoRetryTopicScenarioTests.java | 101 +++++++------ 4 files changed, 320 insertions(+), 163 deletions(-) diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java index c60613fe89..28fe219347 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java @@ -335,8 +335,15 @@ static class FirstTopicListener { @KafkaHandler public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { - container.countDownLatch1.countDown(); - throw new RuntimeException("Woooops... in topic " + receivedTopic); + try { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatch1.countDown(); + } }); } @@ -351,8 +358,15 @@ static class SecondTopicListener { @KafkaHandler public CompletableFuture listenAgain(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatch2); - throw new IllegalStateException("Another woooops... " + receivedTopic); + try { + throw new IllegalStateException("Another woooops... " + receivedTopic); + } + catch (IllegalStateException e){ + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch2); + } }); } } @@ -378,8 +392,15 @@ static class ThirdTopicListener { @KafkaHandler public CompletableFuture listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatch3); - throw new MyRetryException("Annotated woooops... " + receivedTopic); + try { + throw new MyRetryException("Annotated woooops... " + receivedTopic); + } + catch (MyRetryException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch3); + } }); } @@ -401,8 +422,15 @@ static class FourthTopicListener { @KafkaHandler public CompletableFuture listenNoDlt(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatch4); - throw new IllegalStateException("Another woooops... " + receivedTopic); + try { + throw new IllegalStateException("Another woooops... " + receivedTopic); + } + catch (IllegalStateException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch4); + } }); } @@ -447,8 +475,15 @@ static class FifthTopicListener1 extends AbstractFifthTopicListener { public CompletableFuture listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); - throw new RuntimeException("Annotated woooops... " + receivedTopic); + try { + throw new RuntimeException("Annotated woooops... " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); + } }); } @@ -474,8 +509,15 @@ static class FifthTopicListener2 extends AbstractFifthTopicListener { public CompletableFuture listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.countDownLatch52.countDown(); - throw new RuntimeException("Annotated woooops... " + receivedTopic); + try { + throw new RuntimeException("Annotated woooops... " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatch52.countDown(); + } }); } @@ -500,8 +542,15 @@ public CompletableFuture listenNoDlt( @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic, @SuppressWarnings("unused") Acknowledgment ack) { return CompletableFuture.supplyAsync(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatch6); - throw new IllegalStateException("Another woooops... " + receivedTopic); + try { + throw new IllegalStateException("Another woooops... " + receivedTopic); + } + catch (IllegalStateException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch6); + } }); } @@ -523,8 +572,15 @@ static class NoRetryTopicListener { @KafkaHandler public CompletableFuture listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return CompletableFuture.supplyAsync(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatchNoRetry); - throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); + try { + throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); + } + catch (MyDontRetryException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatchNoRetry); + } }); } @@ -552,8 +608,15 @@ static class FirstReuseRetryTopicListener { public CompletableFuture listen1(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.countDownLatchReuseOne.countDown(); - throw new RuntimeException("Another woooops... " + receivedTopic); + try { + throw new RuntimeException("Another woooops... " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatchReuseOne.countDown(); + } }); } @@ -577,8 +640,15 @@ static class SecondReuseRetryTopicListener { public CompletableFuture listen2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.countDownLatchReuseTwo.countDown(); - throw new RuntimeException("Another woooops... " + receivedTopic); + try { + throw new RuntimeException("Another woooops... " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatchReuseTwo.countDown(); + } }); } @@ -598,8 +668,15 @@ static class ThirdReuseRetryTopicListener { public CompletableFuture listen3(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.countDownLatchReuseThree.countDown(); - throw new RuntimeException("Another woooops... " + receivedTopic); + try { + throw new RuntimeException("Another woooops... " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatchReuseThree.countDown(); + } }); } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java index 1dd852e121..6e70802f78 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -58,7 +58,6 @@ import org.springframework.messaging.converter.SmartMessageConverter; import org.springframework.messaging.handler.annotation.Header; import org.springframework.scheduling.TaskScheduler; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; @@ -103,8 +102,7 @@ public class AsyncCompletableFutureRetryTopicScenarioTests { @Test void allFailCaseTest( @Autowired TestTopicListener0 zeroTopicListener, - @Autowired MyCustomDltProcessor myCustomDltProcessor0, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor0) { // All Fail case. String shortFailedMsg1 = "0"; String shortFailedMsg2 = "1"; @@ -172,8 +170,7 @@ void allFailCaseTest( @Test void firstShortFailAndLastLongSuccessRetryTest( @Autowired TestTopicListener1 testTopicListener1, - @Autowired MyCustomDltProcessor myCustomDltProcessor1, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor1) { // Given String longSuccessMsg = "3"; String shortFailedMsg = "1"; @@ -221,8 +218,7 @@ void firstShortFailAndLastLongSuccessRetryTest( @Test void firstLongSuccessAndLastShortFailed( @Autowired TestTopicListener2 zero2TopicListener, - @Autowired MyCustomDltProcessor myCustomDltProcessor2, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor2) { // Given String shortFailedMsg = "1"; String longSuccessMsg = "3"; @@ -270,8 +266,7 @@ void firstLongSuccessAndLastShortFailed( @Test void longFailMsgTwiceThenShortSucessMsgThird( @Autowired TestTopicListener3 testTopicListener3, - @Autowired MyCustomDltProcessor myCustomDltProcessor3, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor3) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("3-topicId", TEST_TOPIC3); @@ -334,8 +329,7 @@ void longFailMsgTwiceThenShortSucessMsgThird( @Test void longSuccessMsgTwiceThenShortFailMsgTwice( @Autowired TestTopicListener4 topicListener4, - @Autowired MyCustomDltProcessor myCustomDltProcessor4, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor4) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("4-TopicId", TEST_TOPIC4); @@ -393,10 +387,9 @@ void longSuccessMsgTwiceThenShortFailMsgTwice( } @Test - void oneLongSuccessMsgBetweenHunderedShortFailMsg( + void oneLongSuccessMsgBetween100ShortFailMsg( @Autowired TestTopicListener5 topicListener5, - @Autowired MyCustomDltProcessor myCustomDltProcessor5, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor5) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("5-TopicId", TEST_TOPIC5); @@ -446,8 +439,7 @@ void oneLongSuccessMsgBetweenHunderedShortFailMsg( @Test void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime( @Autowired TestTopicListener6 topicListener6, - @Autowired MyCustomDltProcessor myCustomDltProcessor6, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor6) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("6-TopicId", TEST_TOPIC6); @@ -548,9 +540,16 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEI this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { - container.countDownLatch0.countDown(); - throw new RuntimeException("Woooops... in topic " + receivedTopic); - }); + try { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatch0.countDown(); + } + }, CompletableFuture.delayedExecutor(Integer.parseInt(message), TimeUnit.MILLISECONDS)); } } @@ -577,19 +576,19 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC this.receivedTopics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { try { - Thread.sleep(Integer.parseInt(message)); + if (message.equals("1")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } - catch (InterruptedException e) { - throw new RuntimeException(e); + catch (RuntimeException e) { + throw e; } finally { container.countDownLatch1.countDown(); } - if (message.equals("1")) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } + return "Task Completed"; - }); + }, CompletableFuture.delayedExecutor(Integer.parseInt(message), TimeUnit.MILLISECONDS)); } } @@ -617,20 +616,19 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC return CompletableFuture.supplyAsync(() -> { try { - Thread.sleep(Integer.parseInt(message)); + if (message.equals("1")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } - catch (InterruptedException e) { - throw new RuntimeException(e); + catch (RuntimeException e) { + throw e; } finally { container.countDownLatch2.countDown(); } - if (message.equals("1")) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } return "Task Completed"; - }); + }, CompletableFuture.delayedExecutor(Integer.parseInt(message), TimeUnit.MILLISECONDS)); } } @@ -650,7 +648,7 @@ static class TestTopicListener3 { private final List receivedTopics = new ArrayList<>(); - public static final String LONG_FAIL_MSG = "5000"; + public static final String LONG_FAIL_MSG = "100"; public static final String SHORT_SUCCESS_MSG = "1"; @@ -661,20 +659,19 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC return CompletableFuture.supplyAsync(() -> { try { - Thread.sleep(Integer.parseInt(message)); + if (message.equals(LONG_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } - catch (InterruptedException e) { - throw new RuntimeException(e); + catch (RuntimeException e) { + throw e; } finally { container.countDownLatch3.countDown(); } - if (message.equals(LONG_FAIL_MSG)) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } return "Task Completed"; - }); + }, CompletableFuture.delayedExecutor(Integer.parseInt(message), TimeUnit.MILLISECONDS)); } } @@ -694,31 +691,30 @@ static class TestTopicListener4 { private final List receivedTopics = new ArrayList<>(); - public static final String LONG_SUCCESS_MSG = "5000"; + public static final String LONG_SUCCESS_MSG = "100"; public static final String SHORT_FAIL_MSG = "1"; @KafkaHandler - public CompletableFuture listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { + public CompletableFuture listen(String message, + @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); return CompletableFuture.supplyAsync(() -> { try { - Thread.sleep(Integer.parseInt(message)); + if (message.equals(SHORT_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } - catch (InterruptedException e) { - throw new RuntimeException(e); + catch (RuntimeException e) { + throw e; } finally { container.countDownLatch4.countDown(); } - - if (message.equals(SHORT_FAIL_MSG)) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } return "Task Completed"; - }); + }, CompletableFuture.delayedExecutor(Integer.parseInt(message), TimeUnit.MILLISECONDS)); } } @@ -738,7 +734,7 @@ static class TestTopicListener5 { private final List receivedTopics = new ArrayList<>(); - public static final String LONG_SUCCESS_MSG = "5000"; + public static final String LONG_SUCCESS_MSG = "100"; public static final String SHORT_FAIL_MSG = "1"; @@ -749,20 +745,19 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC return CompletableFuture.supplyAsync(() -> { try { - Thread.sleep(Integer.parseInt(message)); + if (message.equals(SHORT_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } - catch (InterruptedException e) { - throw new RuntimeException(e); + catch (RuntimeException e) { + throw e; } finally { container.countDownLatch5.countDown(); } - if (message.startsWith(SHORT_FAIL_MSG)) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } return "Task Completed"; - }); + }, CompletableFuture.delayedExecutor(Integer.parseInt(message), TimeUnit.MILLISECONDS)); } } @@ -791,26 +786,25 @@ public CompletableFuture listen(String message, @Header(KafkaHeaders.REC this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); - return CompletableFuture.supplyAsync(() -> { - String[] split = message.split(","); - String sleepAWhile = split[0]; - String failOrSuccess = split[1]; + String[] split = message.split(","); + String sleepAWhile = split[0]; + String failOrSuccess = split[1]; + return CompletableFuture.supplyAsync(() -> { try { - Thread.sleep(Integer.parseInt(sleepAWhile)); + if (failOrSuccess.equals("f")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } - catch (InterruptedException e) { - throw new RuntimeException(e); + catch (RuntimeException e) { + throw e; } finally { container.countDownLatch6.countDown(); } - if (failOrSuccess.equals("f")) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } return "Task Completed"; - }); + }, CompletableFuture.delayedExecutor(Integer.parseInt(sleepAWhile), TimeUnit.MILLISECONDS)); } } @@ -1169,11 +1163,6 @@ TaskScheduler sched() { return new ThreadPoolTaskScheduler(); } - @Bean - ThreadPoolTaskExecutor threadPoolTaskExecutor() { - return new ThreadPoolTaskExecutor(); - } - } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java index 6db3e938f4..0ff9a7bb03 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java @@ -335,8 +335,15 @@ public Mono listen( String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { - container.countDownLatch1.countDown(); - throw new RuntimeException("Woooops... in topic " + receivedTopic); + try { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatch1.countDown(); + } }).then(); } @@ -353,8 +360,15 @@ public Mono listenAgain( String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatch2); - throw new IllegalStateException("Another woooops... " + receivedTopic); + try { + throw new IllegalStateException("Another woooops... " + receivedTopic); + } + catch (IllegalStateException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch2); + } }).then(); } } @@ -382,8 +396,15 @@ public Mono listenWithAnnotation( String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatch3); - throw new MyRetryException("Annotated woooops... " + receivedTopic); + try { + throw new MyRetryException("Annotated woooops... " + receivedTopic); + } + catch (MyRetryException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch3); + } }).then(); } @@ -407,8 +428,15 @@ public Mono listenNoDlt( String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatch4); - throw new IllegalStateException("Another woooops... " + receivedTopic); + try { + throw new IllegalStateException("Another woooops... " + receivedTopic); + } + catch (IllegalStateException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch4); + } }).then(); } @@ -453,8 +481,15 @@ static class FifthTopicListener1 extends AbstractFifthTopicListener { public Mono listenWithAnnotation(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); return Mono.fromCallable(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); - throw new RuntimeException("Annotated woooops... " + receivedTopic); + try { + throw new RuntimeException("Annotated woooops... " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch51); + } }).then(); } @@ -480,8 +515,15 @@ static class FifthTopicListener2 extends AbstractFifthTopicListener { public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); return Mono.fromCallable(() -> { - container.countDownLatch52.countDown(); - throw new RuntimeException("Annotated woooops... " + receivedTopic); + try { + throw new RuntimeException("Annotated woooops... " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatch52.countDown(); + } }).then(); } @@ -506,8 +548,16 @@ public Mono listenNoDlt( @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic, @SuppressWarnings("unused") Acknowledgment ack) { return Mono.fromCallable(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatch6); - throw new IllegalStateException("Another woooops... " + receivedTopic); + try { + throw new IllegalStateException("Another woooops... " + receivedTopic); + + } + catch (IllegalStateException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatch6); + } }).then(); } @@ -529,8 +579,15 @@ static class NoRetryTopicListener { @KafkaHandler public Mono listenWithAnnotation2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { return Mono.fromCallable(() -> { - container.countDownIfNotKnown(receivedTopic, container.countDownLatchNoRetry); - throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); + try { + throw new MyDontRetryException("Annotated second woooops... " + receivedTopic); + } + catch (MyDontRetryException e) { + throw e; + } + finally { + container.countDownIfNotKnown(receivedTopic, container.countDownLatchNoRetry); + } }).then(); } @@ -558,8 +615,15 @@ static class FirstReuseRetryTopicListener { public Mono listen1(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); return Mono.fromCallable(() -> { - container.countDownLatchReuseOne.countDown(); - throw new RuntimeException("Another woooops... " + receivedTopic); + try { + throw new RuntimeException("Another woooops... " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatchReuseOne.countDown(); + } }).then(); } @@ -583,8 +647,15 @@ static class SecondReuseRetryTopicListener { public Mono listen2(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); return Mono.fromCallable(() -> { - container.countDownLatchReuseTwo.countDown(); - throw new RuntimeException("Another woooops... " + receivedTopic); + try { + throw new RuntimeException("Another woooops... " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatchReuseTwo.countDown(); + } }).then(); } @@ -604,8 +675,15 @@ static class ThirdReuseRetryTopicListener { public Mono listen3(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic) { this.topics.add(receivedTopic); return Mono.fromCallable(() -> { - container.countDownLatchReuseThree.countDown(); - throw new RuntimeException("Another woooops... " + receivedTopic); + try { + throw new RuntimeException("Another woooops... " + receivedTopic); + } + catch (RuntimeException e) { + throw e; + } + finally { + container.countDownLatchReuseThree.countDown(); + } }).then(); } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java index 2392abc6d4..f9f3f01d88 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -57,7 +57,6 @@ import org.springframework.messaging.converter.SmartMessageConverter; import org.springframework.messaging.handler.annotation.Header; import org.springframework.scheduling.TaskScheduler; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; @@ -104,8 +103,7 @@ public class AsyncMonoRetryTopicScenarioTests { @Test void allFailCaseTest( @Autowired TestTopicListener0 zeroTopicListener, - @Autowired MyCustomDltProcessor myCustomDltProcessor0, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor0) { // Given String shortFailedMsg1 = "0"; String shortFailedMsg2 = "1"; @@ -173,8 +171,7 @@ void allFailCaseTest( @Test void firstShortFailAndLastLongSuccessRetryTest( @Autowired TestTopicListener1 testTopicListener1, - @Autowired MyCustomDltProcessor myCustomDltProcessor1, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor1) { // Given String longSuccessMsg = "3"; String shortFailedMsg = "1"; @@ -221,8 +218,7 @@ void firstShortFailAndLastLongSuccessRetryTest( @Test void firstLongSuccessAndLastShortFailed( @Autowired TestTopicListener2 zero2TopicListener, - @Autowired MyCustomDltProcessor myCustomDltProcessor2, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor2) { // Given String shortFailedMsg = "1"; String longSuccessMsg = "3"; @@ -269,8 +265,7 @@ void firstLongSuccessAndLastShortFailed( @Test void longFailMsgTwiceThenShortSuccessMsgThird( @Autowired TestTopicListener3 testTopicListener3, - @Autowired MyCustomDltProcessor myCustomDltProcessor3, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor3) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("3-topicId", TEST_TOPIC3); @@ -333,8 +328,7 @@ void longFailMsgTwiceThenShortSuccessMsgThird( @Test void longSuccessMsgTwiceThenShortFailMsgTwice( @Autowired TestTopicListener4 topicListener4, - @Autowired MyCustomDltProcessor myCustomDltProcessor4, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor4) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("4-TopicId", TEST_TOPIC4); @@ -392,10 +386,9 @@ void longSuccessMsgTwiceThenShortFailMsgTwice( } @Test - void oneLongSuccessMsgBetweenHunderedShortFailMsg( + void oneLongSuccessMsgBetween100ShortFailMsgs( @Autowired TestTopicListener5 topicListener5, - @Autowired MyCustomDltProcessor myCustomDltProcessor5, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor5) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("5-TopicId", TEST_TOPIC5); @@ -444,8 +437,7 @@ void oneLongSuccessMsgBetweenHunderedShortFailMsg( @Test void halfSuccessMsgAndHalfFailedMsgWithRandomSleepTime( @Autowired TestTopicListener6 topicListener6, - @Autowired MyCustomDltProcessor myCustomDltProcessor6, - @Autowired ThreadPoolTaskExecutor executor) { + @Autowired MyCustomDltProcessor myCustomDltProcessor6) { // Given DestinationTopic destinationTopic = topicContainer.getNextDestinationTopicFor("6-TopicId", TEST_TOPIC6); @@ -545,8 +537,15 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) St this.receivedMsgs.add(message); this.receivedTopics.add(receivedTopic); return Mono.fromCallable(() -> { - container.countDownLatch0.countDown(); - throw new RuntimeException("Woooops... in topic " + receivedTopic); + try { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } + catch (Exception e) { + throw e; + } + finally { + container.countDownLatch0.countDown(); + } }).then(); } @@ -576,16 +575,20 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) return Mono.fromCallable(() -> { try { Thread.sleep(Integer.parseInt(message)); + if (message.equals("1")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } catch (InterruptedException e) { throw new RuntimeException(e); } + catch (RuntimeException e) { + throw e; + } finally { container.countDownLatch1.countDown(); } - if (message.equals("1")) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } + return "Task Completed"; }); @@ -616,17 +619,20 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) return Mono.fromCallable(() -> { try { Thread.sleep(Integer.parseInt(message)); + if (message.equals("1")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } catch (InterruptedException e) { throw new RuntimeException(e); } + catch (RuntimeException e) { + throw e; + } finally { container.countDownLatch2.countDown(); } - if (message.equals("1")) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } return "Task Completed"; }); } @@ -648,7 +654,7 @@ static class TestTopicListener3 { private final List receivedTopics = new ArrayList<>(); - public static final String LONG_FAIL_MSG = "5000"; + public static final String LONG_FAIL_MSG = "100"; public static final String SHORT_SUCCESS_MSG = "1"; @@ -660,17 +666,20 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) return Mono.fromCallable(() -> { try { Thread.sleep(Integer.parseInt(message)); + if (message.equals(LONG_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } catch (InterruptedException e) { throw new RuntimeException(e); } + catch (RuntimeException e) { + throw e; + } finally { container.countDownLatch3.countDown(); } - if (message.equals(LONG_FAIL_MSG)) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } return "Task Completed"; }); @@ -694,7 +703,7 @@ static class TestTopicListener4 { private final List receivedTopics = new ArrayList<>(); - public static final String LONG_SUCCESS_MSG = "5000"; + public static final String LONG_SUCCESS_MSG = "100"; public static final String SHORT_FAIL_MSG = "1"; @@ -706,17 +715,20 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) return Mono.fromCallable(() -> { try { Thread.sleep(Integer.parseInt(message)); + if (message.equals(SHORT_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } catch (InterruptedException e) { throw new RuntimeException(e); } + catch (RuntimeException e) { + throw e; + } finally { container.countDownLatch4.countDown(); } - if (message.equals(SHORT_FAIL_MSG)) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } return "Task Completed"; }); @@ -739,7 +751,7 @@ static class TestTopicListener5 { private final List receivedTopics = new ArrayList<>(); - public static final String LONG_SUCCESS_MSG = "5000"; + public static final String LONG_SUCCESS_MSG = "100"; public static final String SHORT_FAIL_MSG = "1"; @@ -751,17 +763,20 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) return Mono.fromCallable(() -> { try { Thread.sleep(Integer.parseInt(message)); + if (message.equals(SHORT_FAIL_MSG)) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } catch (InterruptedException e) { throw new RuntimeException(e); } + catch (RuntimeException e) { + throw e; + } finally { container.countDownLatch5.countDown(); } - if (message.startsWith(SHORT_FAIL_MSG)) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } return "Task Completed"; }); } @@ -799,17 +814,20 @@ public Mono listen(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) try { Thread.sleep(Integer.parseInt(sleepAWhile)); + if (failOrSuccess.equals("f")) { + throw new RuntimeException("Woooops... in topic " + receivedTopic); + } } catch (InterruptedException e) { throw new RuntimeException(e); } + catch (RuntimeException e) { + throw e; + } finally { container.countDownLatch6.countDown(); } - if (failOrSuccess.equals("f")) { - throw new RuntimeException("Woooops... in topic " + receivedTopic); - } return "Task Completed"; }); } @@ -1168,11 +1186,6 @@ TaskScheduler sched() { return new ThreadPoolTaskScheduler(); } - @Bean - ThreadPoolTaskExecutor threadPoolTaskExecutor() { - return new ThreadPoolTaskExecutor(); - } - } } From 1d9f7b92b65bea92a14197e924928a35c216e900 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 15 Oct 2024 09:46:26 +0900 Subject: [PATCH 23/30] Remove weed from async retry test and fix lint error. --- ...FutureRetryTopicClassLevelIntegrationTests.java | 14 +------------- ...FutureRetryTopicClassLevelIntegrationTests.java | 12 ------------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java index 28fe219347..a0cbf2de06 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicClassLevelIntegrationTests.java @@ -23,7 +23,6 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -32,7 +31,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.apache.kafka.clients.admin.AdminClientConfig; @@ -180,16 +178,6 @@ void shouldRetryThirdTopicWithTimeout( m.setAccessible(true); method.set(m); }, m -> m.getName().equals("newTopics")); - @SuppressWarnings("unchecked") - Collection weededTopics = (Collection) method.get().invoke(admin); - AtomicInteger weeded = new AtomicInteger(); - weededTopics.forEach(topic -> { - if (topic.name().equals(THIRD_TOPIC) || topic.name().equals(FOURTH_TOPIC)) { - assertThat(topic).isExactlyInstanceOf(NewTopic.class); - weeded.incrementAndGet(); - } - }); - assertThat(weeded.get()).isEqualTo(2); registry.getListenerContainerIds().stream() .filter(id -> id.startsWith("third")) .forEach(id -> { @@ -361,7 +349,7 @@ public CompletableFuture listenAgain(String message, @Header(KafkaHeaders. try { throw new IllegalStateException("Another woooops... " + receivedTopic); } - catch (IllegalStateException e){ + catch (IllegalStateException e) { throw e; } finally { diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java index 0ff9a7bb03..469de356df 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoFutureRetryTopicClassLevelIntegrationTests.java @@ -23,7 +23,6 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -31,7 +30,6 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.apache.kafka.clients.admin.AdminClientConfig; @@ -180,16 +178,6 @@ void shouldRetryThirdTopicWithTimeout( m.setAccessible(true); method.set(m); }, m -> m.getName().equals("newTopics")); - @SuppressWarnings("unchecked") - Collection weededTopics = (Collection) method.get().invoke(admin); - AtomicInteger weeded = new AtomicInteger(); - weededTopics.forEach(topic -> { - if (topic.name().equals(THIRD_TOPIC) || topic.name().equals(FOURTH_TOPIC)) { - assertThat(topic).isExactlyInstanceOf(NewTopic.class); - weeded.incrementAndGet(); - } - }); - assertThat(weeded.get()).isEqualTo(2); registry.getListenerContainerIds().stream() .filter(id -> id.startsWith("third")) .forEach(id -> { From e418b10399d00c2976cf741b4dcb171cbe72da75 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 15 Oct 2024 09:51:30 +0900 Subject: [PATCH 24/30] Add Tags for conditional test. --- .../AsyncCompletableFutureRetryTopicScenarioTests.java | 2 ++ .../kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java index 6e70802f78..9364794a2a 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -31,6 +31,7 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -68,6 +69,7 @@ * @since 3.3.0 */ +@Tag("async-retry-flaky-test") @SpringJUnitConfig @DirtiesContext @EmbeddedKafka diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java index f9f3f01d88..d9ffddd3ec 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -30,6 +30,7 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -69,6 +70,7 @@ * @since 3.3.0 */ +@Tag("async-retry-flaky-test") @SpringJUnitConfig @DirtiesContext @EmbeddedKafka From 002aaf4cf5b501ac5998d3b8ef1b625ac9c97351 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 15 Oct 2024 21:57:15 +0900 Subject: [PATCH 25/30] Remove useless tag for async failure retry tests. --- .../AsyncCompletableFutureRetryTopicScenarioTests.java | 1 - .../kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java | 1 - 2 files changed, 2 deletions(-) diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java index 9364794a2a..d4a5715339 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -69,7 +69,6 @@ * @since 3.3.0 */ -@Tag("async-retry-flaky-test") @SpringJUnitConfig @DirtiesContext @EmbeddedKafka diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java index d9ffddd3ec..33413781c7 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -70,7 +70,6 @@ * @since 3.3.0 */ -@Tag("async-retry-flaky-test") @SpringJUnitConfig @DirtiesContext @EmbeddedKafka From 3b79d59dec17bcfb1ca1325a3634dd23a68f612d Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 15 Oct 2024 22:04:13 +0900 Subject: [PATCH 26/30] Remove unused import --- .../AsyncCompletableFutureRetryTopicScenarioTests.java | 1 - .../kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java | 1 - 2 files changed, 2 deletions(-) diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java index d4a5715339..6e70802f78 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -31,7 +31,6 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java index 33413781c7..f9f3f01d88 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -30,7 +30,6 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; From 82ecac7c9cac1f913f440650921c6a269a2292b9 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 15 Oct 2024 23:37:12 +0900 Subject: [PATCH 27/30] Make a method static and Add @SuppressWarning. --- .../listener/adapter/MessagingMessageListenerAdapter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java index 07fbac0030..9fc07d6827 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/adapter/MessagingMessageListenerAdapter.java @@ -681,14 +681,15 @@ protected void asyncFailure(Object request, @Nullable Acknowledgment acknowledgm acknowledge(acknowledgment); if (canAsyncRetry(request, ex) && Objects.nonNull(this.callbackForAsyncFailure)) { + @SuppressWarnings("unchecked") ConsumerRecord record = (ConsumerRecord) request; this.callbackForAsyncFailure.accept(record, (RuntimeException) ex); } } } - private boolean canAsyncRetry(Object request, Throwable exception) { - // The async retry with @RetryableTopic only support in case of SingleRecord Listener. + private static boolean canAsyncRetry(Object request, Throwable exception) { + // The async retry with @RetryableTopic is only supported for SingleRecord Listener. return request instanceof ConsumerRecord && exception instanceof RuntimeException; } From 0afae4edb4146b8998413f9c73652830af2b57c2 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Wed, 16 Oct 2024 00:03:52 +0900 Subject: [PATCH 28/30] Remove useless latch. --- .../AsyncCompletableFutureRetryTopicScenarioTests.java | 6 ------ .../kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java | 6 ------ 2 files changed, 12 deletions(-) diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java index 6e70802f78..0bb52e0d63 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncCompletableFutureRetryTopicScenarioTests.java @@ -866,10 +866,6 @@ static class CountDownLatchContainer { CountDownLatch dltCountdownLatch6 = new CountDownLatch(DLT_COUNT6); - CountDownLatch customErrorHandlerCountdownLatch = new CountDownLatch(6); - - CountDownLatch customMessageConverterCountdownLatch = new CountDownLatch(6); - } static class MyCustomDltProcessor { @@ -973,7 +969,6 @@ RetryTopicConfiguration testRetryTopic6(KafkaTemplate template) KafkaListenerErrorHandler myCustomErrorHandler( CountDownLatchContainer container) { return (message, exception) -> { - container.customErrorHandlerCountdownLatch.countDown(); throw exception; }; } @@ -985,7 +980,6 @@ SmartMessageConverter myCustomMessageConverter( @Override public Object fromMessage(Message message, Class targetClass, Object conversionHint) { - container.customMessageConverterCountdownLatch.countDown(); return super.fromMessage(message, targetClass, conversionHint); } }; diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java index f9f3f01d88..ed467cf23c 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AsyncMonoRetryTopicScenarioTests.java @@ -891,10 +891,6 @@ static class CountDownLatchContainer { CountDownLatch dltCountdownLatch6 = new CountDownLatch(DLT_COUNT6); - CountDownLatch customErrorHandlerCountdownLatch = new CountDownLatch(6); - - CountDownLatch customMessageConverterCountdownLatch = new CountDownLatch(6); - } static class MyCustomDltProcessor { @@ -997,7 +993,6 @@ RetryTopicConfiguration testRetryTopic6(KafkaTemplate template) KafkaListenerErrorHandler myCustomErrorHandler( CountDownLatchContainer container) { return (message, exception) -> { - container.customErrorHandlerCountdownLatch.countDown(); throw exception; }; } @@ -1009,7 +1004,6 @@ SmartMessageConverter myCustomMessageConverter( @Override public Object fromMessage(Message message, Class targetClass, Object conversionHint) { - container.customMessageConverterCountdownLatch.countDown(); return super.fromMessage(message, targetClass, conversionHint); } }; From 796397f6a329eec9da5b6f966a2dd17debc59236 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Wed, 16 Oct 2024 08:25:45 +0900 Subject: [PATCH 29/30] Use specific ListenerAdapter type for async retry feature. --- .../kafka/listener/KafkaMessageListenerContainer.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index 66f623fc4d..595c49a63f 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -109,6 +109,7 @@ import org.springframework.kafka.listener.ContainerProperties.EOSMode; import org.springframework.kafka.listener.adapter.AbstractDelegatingMessageListenerAdapter; import org.springframework.kafka.listener.adapter.AsyncRepliesAware; +import org.springframework.kafka.listener.adapter.KafkaBackoffAwareMessageListenerAdapter; import org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter; import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.KafkaHeaders; @@ -904,10 +905,11 @@ else if (listener instanceof MessageListener) { this.observationEnabled = this.containerProperties.isObservationEnabled(); if (!AopUtils.isAopProxy(this.genericListener) && - this.genericListener instanceof AbstractDelegatingMessageListenerAdapter) { - AbstractDelegatingMessageListenerAdapter> genListener = - (AbstractDelegatingMessageListenerAdapter>) this.genericListener; + this.genericListener instanceof KafkaBackoffAwareMessageListenerAdapter) { + KafkaBackoffAwareMessageListenerAdapter genListener = + (KafkaBackoffAwareMessageListenerAdapter) this.genericListener; if (genListener.getDelegate() instanceof RecordMessagingMessageListenerAdapter adapterListener) { + // This means that the async retry feature is supported only for SingleRecordListener with @RetryableTopic. adapterListener.setCallbackForAsyncFailure(this::callbackForAsyncFailure); } } From 4e89d535c4057d921351491dc52178e9e75d88d7 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Wed, 16 Oct 2024 08:30:14 +0900 Subject: [PATCH 30/30] Fixes lint error. --- .../kafka/listener/KafkaMessageListenerContainer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index 595c49a63f..1faaf16379 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -107,7 +107,6 @@ import org.springframework.kafka.listener.ContainerProperties.AckMode; import org.springframework.kafka.listener.ContainerProperties.AssignmentCommitOption; import org.springframework.kafka.listener.ContainerProperties.EOSMode; -import org.springframework.kafka.listener.adapter.AbstractDelegatingMessageListenerAdapter; import org.springframework.kafka.listener.adapter.AsyncRepliesAware; import org.springframework.kafka.listener.adapter.KafkaBackoffAwareMessageListenerAdapter; import org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter; @@ -905,7 +904,7 @@ else if (listener instanceof MessageListener) { this.observationEnabled = this.containerProperties.isObservationEnabled(); if (!AopUtils.isAopProxy(this.genericListener) && - this.genericListener instanceof KafkaBackoffAwareMessageListenerAdapter) { + this.genericListener instanceof KafkaBackoffAwareMessageListenerAdapter) { KafkaBackoffAwareMessageListenerAdapter genListener = (KafkaBackoffAwareMessageListenerAdapter) this.genericListener; if (genListener.getDelegate() instanceof RecordMessagingMessageListenerAdapter adapterListener) {