From ac393f2550958c110a5d0aa3022f5dad6366811b Mon Sep 17 00:00:00 2001
From: Jorgen Ringen <jorgen.ringen@ruter.no>
Date: Tue, 10 Nov 2020 23:16:30 +0100
Subject: [PATCH] feature: Choose between Consumer commit or Producer
 transactional commits

- Choose either Consumer sync or async commits
- Fixes #25 https://github.com/confluentinc/parallel-consumer/issues/25:
-- Sometimes a a transaction error occurs - Cannot call send in state COMMITTING_TRANSACTION #25
- ReentrantReadWrite lock protects non-thread safe transactional producer from incorrect multithreaded use
- Wider lock to prevent transaction's containing produced messages that they shouldn't
- Implement non transactional synchronous commit sync properly
- Select tests adapted to non transactional as well
- Must start tx in MockProducer as well
- Adds supervision to poller
- Fixes a performance issue with the async committer not being woken up
- Enhances tests to run under multiple commit modes
- Fixes example app tests - incorrectly testing wrong thing and MockProducer not configured to auto complete
- Make committer thread revoke partitions and commit
- Have onPartitionsRevoked be responsible for committing on close, instead of an explicit call to commit by controller
- Make sure Broker Poller now drains properly, committing any waiting work
- Add missing revoke flow to MockConsumer wrapper
- Add missing latch timeout check
---
 .gitignore                                    |   5 +-
 .idea/runConfigurations/All_examples.xml      |   2 +-
 README.adoc                                   |  59 ++--
 .../io/confluent/csid/utils/StringUtils.java  |   8 +-
 .../AbstractOffsetCommitter.java              |  59 ++++
 .../parallelconsumer/BrokerPollSystem.java    | 192 +++++++----
 .../parallelconsumer/ConsumerManager.java     | 114 +++++++
 .../ConsumerOffsetCommitter.java              | 203 ++++++++++++
 .../ErrorInUserFunctionException.java         |  10 +
 .../parallelconsumer/InternalError.java       |  20 ++
 .../parallelconsumer/OffsetCommitter.java     |   5 +
 .../ParallelConsumerOptions.java              |  82 ++++-
 .../ParallelEoSStreamProcessor.java           | 313 ++++++++----------
 .../parallelconsumer/ProducerManager.java     | 311 +++++++++++++++++
 .../parallelconsumer/UserFunctions.java       |  38 +++
 .../parallelconsumer/WorkManager.java         |  53 +--
 .../CloseAndOpenOffsetTest.java               |  85 ++++-
 .../integrationTests/KafkaTest.java           |  36 +-
 .../integrationTests/LoadTest.java            |   6 +-
 .../TransactionAndCommitModeTest.java         | 143 ++++++++
 .../integrationTests/VolumeTests.java         | 116 ++++---
 .../utils/KafkaClientUtils.java               |   7 +-
 .../confluent/csid/utils/KafkaTestUtils.java  |  44 ++-
 .../csid/utils/LongPollingMockConsumer.java   |  92 ++++-
 .../ParallelEoSStreamProcessorTest.java       | 176 ++++++----
 .../ParallelEoSStreamProcessorTestBase.java   | 142 ++++++--
 .../parallelconsumer/WorkManagerTest.java     |   6 +-
 .../test/resources/junit-platform.properties  |   2 +-
 .../src/test/resources/logback-test.xml       |  23 +-
 .../examples/core/CoreApp.java                |  50 +--
 .../examples/core/CoreAppTest.java            |  31 +-
 ...logback-test.xml => logback-temp-test.xml} |   2 +-
 .../examples/streams/StreamsApp.java          |  11 +-
 .../examples/streams/StreamsAppTest.java      |   2 +-
 .../parallel-consumer-example-vertx/pom.xml   |  13 +
 .../examples/vertx/VertxApp.java              |  12 +-
 .../examples/vertx/VertxAppTest.java          |  29 +-
 ...StreamVertxParallelEoSStreamProcessor.java |  10 +-
 .../VertxParallelEoSStreamProcessor.java      |  16 +-
 .../parallelconsumer/vertx/VertxTest.java     |  12 +-
 src/docs/README.adoc                          |  29 +-
 41 files changed, 2036 insertions(+), 533 deletions(-)
 create mode 100644 parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/AbstractOffsetCommitter.java
 create mode 100644 parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ConsumerManager.java
 create mode 100644 parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ConsumerOffsetCommitter.java
 create mode 100644 parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ErrorInUserFunctionException.java
 create mode 100644 parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/InternalError.java
 create mode 100644 parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/OffsetCommitter.java
 create mode 100644 parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ProducerManager.java
 create mode 100644 parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/UserFunctions.java
 create mode 100644 parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/TransactionAndCommitModeTest.java
 rename parallel-consumer-examples/parallel-consumer-example-core/src/test/resources/{logback-test.xml => logback-temp-test.xml} (94%)

diff --git a/.gitignore b/.gitignore
index f5833c742..05e3ba15b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,4 +29,7 @@ hs_err_pid*
 *.iml
 target/
 .DS_Store
-*.versionsBackup
\ No newline at end of file
+*.versionsBackup
+
+# JENV
+.java-version
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_examples.xml b/.idea/runConfigurations/All_examples.xml
index ae2384feb..8c82d61b5 100644
--- a/.idea/runConfigurations/All_examples.xml
+++ b/.idea/runConfigurations/All_examples.xml
@@ -7,7 +7,7 @@
         <option name="ENABLED" value="true" />
       </pattern>
     </extension>
-    <option name="PACKAGE_NAME" value="io.confluent.parallalconsumer.examples" />
+    <option name="PACKAGE_NAME" value="io.confluent.parallelconsumer.examples" />
     <option name="MAIN_CLASS_NAME" value="" />
     <option name="METHOD_NAME" value="" />
     <option name="TEST_OBJECT" value="package" />
diff --git a/README.adoc b/README.adoc
index 18ce333de..208c0e4dd 100644
--- a/README.adoc
+++ b/README.adoc
@@ -313,18 +313,17 @@ Where `${project.version}` is the version to be used:
 .Setup the client
 [source,java,indent=0]
 ----
-        var options = ParallelConsumerOptions.builder()
-                .ordering(KEY) // <1>
-                .maxConcurrency(1000) // <2>
-                .maxUncommittedMessagesToHandlePerPartition(10000) // <3>
-                .build();
+        ParallelConsumerOptions options = getOptions();
 
         Consumer<String, String> kafkaConsumer = getKafkaConsumer(); // <4>
+        Producer<String, String> kafkaProducer = getKafkaProducer();
+
+        ParallelStreamProcessor<String, String> eosStreamProcessor = ParallelStreamProcessor.createEosStreamProcessor(kafkaConsumer, kafkaProducer, options);
         if (!(kafkaConsumer instanceof MockConsumer)) {
-            kafkaConsumer.subscribe(UniLists.of(inputTopic)); // <5>
+            eosStreamProcessor.subscribe(UniLists.of(inputTopic)); // <5>
         }
 
-        return ParallelStreamProcessor.createEosStreamProcessor(kafkaConsumer, getKafkaProducer(), options);
+        return eosStreamProcessor;
 ----
 <1> Choose your ordering type, `KEY` in this case.
 This ensures maximum concurrency, while ensuring messages are processed and committed in `KEY` order, making sure no offset is committed unless all offsets before it in it's partition, are completed also.
@@ -384,13 +383,12 @@ You can also optionally provide a callback function to be run after the message(
 [source,java,indent=0]
         this.parallelConsumer.pollAndProduce(record -> {
                     var result = processBrokerRecord(record);
-                    ProducerRecord<String, String> produceRecord =
-                            new ProducerRecord<>(outputTopic, "a-key", result.payload);
-                    return UniLists.of(produceRecord);
-                }, consumeProduceResult ->
-                        log.info("Message {} saved to broker at offset {}",
-                                consumeProduceResult.getOut(),
-                                consumeProduceResult.getMeta().offset())
+                    return new ProducerRecord<>(outputTopic, record.key(), result.payload);
+                }, consumeProduceResult -> {
+                    log.debug("Message {} saved to broker at offset {}",
+                            consumeProduceResult.getOut(),
+                            consumeProduceResult.getMeta().offset());
+                }
         );
 
 
@@ -412,7 +410,7 @@ In future versions, we plan to look at supporting other streaming systems like h
         var resultStream = parallelConsumer.vertxHttpReqInfoStream(record -> {
             log.info("Concurrently constructing and returning RequestInfo from record: {}", record);
             Map<String, String> params = UniMaps.of("recordKey", record.key(), "payload", record.value());
-            return new RequestInfo("localhost", "/api", params); // <1>
+            return new RequestInfo("localhost", port, "/api", params); // <1>
         });
 ----
 <1> Simply return an object representing the request, the Vert.x HTTP engine will handle the rest, using it's non-blocking engine
@@ -453,7 +451,7 @@ image::https://lucid.app/publicSegments/view/43f2740c-2a7f-4b7f-909e-434a5bbe3fb
     }
 
     void concurrentProcess() {
-        setupConsumer();
+        setupParallelConsumer();
 
         parallelConsumer.poll(record -> {
             log.info("Concurrently processing a record: {}", record);
@@ -552,16 +550,33 @@ When your function is actually run, a result object will be streamed back to you
 After your operation completes, you can also choose to publish a result message back to Kafka.
 The message publishing metadata can be streamed back to your client code.
 
-== Apache Kafka EoS Transaction Model
+== Commit Mode
+
+The system gives you three choices for how to do offset commits.
+The simplest of the three are the two Consummer commits modes.
+They are of course, `sycnhronous` and `asynchronous` mode.
+The `transactional` mode is explained in the next section.
+
+`Asychornous` mode is faster, as it doesn't block the control loop.
+
+`Sycnhronous` will block the processing loop until a successful commit response is received, however, `asycnhronous` will still be capped by the max processing settings in the `Options` class.
+
+If you're used to using the auto commit mode in the normal Kafka consumer, you can think of the `Asychornous` mode being similar to this.
+We suggest starting with this mode, and it is the default.
 
-The system uses Kafka's Exactly Once Semantics (EoS) system.
+=== Apache Kafka EoS Transaction Model
+
+There is also the option to use Kafka's Exactly Once Semantics (EoS) system.
 This causes all messages produced as a result of processing a message to be committed within a transaction, along with their source offset.
 This means that even under failure, the results will exist exactly once in the Kafka output topic.
-If as a part of
-your processing, you create side effects in other systems, this pertains to the usual idempotency requirements when breaking of EoS Kafka boundaries.
+If as a part of your processing, you create side effects in other systems, this pertains to the usual idempotency requirements when breaking of EoS Kafka boundaries.
+
+NOTE:: As with the `synchronous` processing mode, this will also block the processing loop until a successful transaction completes
 
 CAUTION: This cannot be true for any externally integrated third party system, unless that system is __idempotent__.
 
+For implementations details, see the <<Transactional System Architecture>> section.
+
 [[streams-usage]]
 == Using with Kafka Streams
 
@@ -748,6 +763,10 @@ Instead of the work thread pool count being the degree of concurrency, it is con
 .Vert.x Architecture
 image::https://lucid.app/publicSegments/view/509df410-5997-46be-98e7-ac7f241780b4/image.png[Vert.x Architecture, align="center"]
 
+=== Transactional System Architecture
+
+image::https://lucid.app/publicSegments/view/7480d948-ed7d-4370-a308-8ec12e6b453b/image.png[]
+
 === Offset Map
 
 ==== Storage
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/StringUtils.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/StringUtils.java
index f78ebf8d2..219066bd0 100644
--- a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/StringUtils.java
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/StringUtils.java
@@ -9,7 +9,11 @@
 public class StringUtils {
 
     public static String msg(String s, Object... args) {
-        String message = MessageFormatter.basicArrayFormat(s, args);
-        return message;
+        return MessageFormatter.basicArrayFormat(s, args);
+    }
+
+    public static boolean isBlank(final String property) {
+        if (property == null) return true;
+        else return property.trim().isEmpty(); // isBlank @since 11
     }
 }
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/AbstractOffsetCommitter.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/AbstractOffsetCommitter.java
new file mode 100644
index 000000000..4f0167987
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/AbstractOffsetCommitter.java
@@ -0,0 +1,59 @@
+package io.confluent.parallelconsumer;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+
+import java.util.Map;
+
+@Slf4j
+@RequiredArgsConstructor
+public abstract class AbstractOffsetCommitter<K, V> implements OffsetCommitter {
+
+    protected final ConsumerManager<K, V> consumerMgr;
+    protected final WorkManager<K, V> wm;
+
+    /**
+     * Get offsets from {@link WorkManager} that are ready to commit
+     */
+    @Override
+    public void retrieveOffsetsAndCommit() {
+        log.debug("Commit starting - find completed work to commit offsets");
+        // todo shouldn't be removed until commit succeeds (there's no harm in committing the same offset twice)
+        preAcquireWork();
+        try {
+            Map<TopicPartition, OffsetAndMetadata> offsetsToSend = wm.findCompletedEligibleOffsetsAndRemove();
+            if (offsetsToSend.isEmpty()) {
+                log.trace("No offsets ready");
+            } else {
+                log.debug("Will commit offsets for {} partition(s): {}", offsetsToSend.size(), offsetsToSend);
+                ConsumerGroupMetadata groupMetadata = consumerMgr.groupMetadata();
+
+                log.debug("Begin commit");
+                commitOffsets(offsetsToSend, groupMetadata);
+
+                log.debug("On commit success");
+                onOffsetCommitSuccess(offsetsToSend);
+            }
+        } finally {
+            postCommit();
+        }
+    }
+
+    protected void postCommit() {
+        // default noop
+    }
+
+    protected void preAcquireWork() {
+        // default noop
+    }
+
+    private void onOffsetCommitSuccess(final Map<TopicPartition, OffsetAndMetadata> offsetsToSend) {
+        wm.onOffsetCommitSuccess(offsetsToSend);
+    }
+
+    protected abstract void commitOffsets(final Map<TopicPartition, OffsetAndMetadata> offsetsToSend, final ConsumerGroupMetadata groupMetadata);
+
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/BrokerPollSystem.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/BrokerPollSystem.java
index aab44f9e7..9649c4824 100644
--- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/BrokerPollSystem.java
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/BrokerPollSystem.java
@@ -4,14 +4,13 @@
  * Copyright (C) 2020 Confluent, Inc.
  */
 
+import io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecords;
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.errors.WakeupException;
-import pl.tlinkowski.unij.api.UniMaps;
 
 import java.time.Duration;
 import java.util.Optional;
@@ -20,8 +19,11 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
 
 import static io.confluent.csid.utils.BackportUtils.toSeconds;
+import static io.confluent.parallelconsumer.ParallelEoSStreamProcessor.State.closed;
+import static io.confluent.parallelconsumer.ParallelEoSStreamProcessor.State.running;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 /**
@@ -31,11 +33,11 @@
  * @param <V>
  */
 @Slf4j
-public class BrokerPollSystem<K, V> {
+public class BrokerPollSystem<K, V> implements OffsetCommitter {
 
-    private final org.apache.kafka.clients.consumer.Consumer<K, V> consumer;
+    private final ConsumerManager<K, V> consumerManager;
 
-    public ParallelEoSStreamProcessor.State state = ParallelEoSStreamProcessor.State.running;
+    private ParallelEoSStreamProcessor.State state = ParallelEoSStreamProcessor.State.running;
 
     private Optional<Future<Boolean>> pollControlThreadFuture;
 
@@ -43,20 +45,47 @@ public class BrokerPollSystem<K, V> {
 
     private final ParallelEoSStreamProcessor<K, V> pc;
 
+    private Optional<ConsumerOffsetCommitter<K, V>> committer = Optional.empty();
+
+    /**
+     * Note how this relates to {@link BrokerPollSystem#getLongPollTimeout()} - if longPollTimeout is high and loading
+     * factor is low, there may not be enough messages queued up to satisfy demand.
+     */
     @Setter
     @Getter
     private static Duration longPollTimeout = Duration.ofMillis(2000);
 
     private final WorkManager<K, V> wm;
 
-    public BrokerPollSystem(Consumer<K, V> consumer, WorkManager<K, V> wm, ParallelEoSStreamProcessor<K, V> pc) {
-        this.consumer = consumer;
+    public BrokerPollSystem(ConsumerManager<K, V> consumerMgr, WorkManager<K, V> wm, ParallelEoSStreamProcessor<K, V> pc, final ParallelConsumerOptions options) {
         this.wm = wm;
         this.pc = pc;
+
+        this.consumerManager = consumerMgr;
+        switch (options.getCommitMode()) {
+            case CONSUMER_SYNC, CONSUMER_ASYNCHRONOUS -> {
+                ConsumerOffsetCommitter<K, V> consumerCommitter = new ConsumerOffsetCommitter<>(consumerMgr, wm, options);
+                committer = Optional.of(consumerCommitter);
+            }
+        }
     }
 
     public void start() {
-        this.pollControlThreadFuture = Optional.of(Executors.newSingleThreadExecutor().submit(this::controlLoop));
+        Future<Boolean> submit = Executors.newSingleThreadExecutor().submit(this::controlLoop);
+        this.pollControlThreadFuture = Optional.of(submit);
+    }
+
+    public void supervise() {
+        if (pollControlThreadFuture.isPresent()) {
+            Future<Boolean> booleanFuture = pollControlThreadFuture.get();
+            if (booleanFuture.isCancelled() || booleanFuture.isDone()) {
+                try {
+                    booleanFuture.get();
+                } catch (Exception e) {
+                    throw new InternalError("Error in " + BrokerPollSystem.class.getSimpleName() + " system.", e);
+                }
+            }
+        }
     }
 
     /**
@@ -65,42 +94,70 @@ public void start() {
     private boolean controlLoop() {
         Thread.currentThread().setName("broker-poll");
         log.trace("Broker poll control loop start");
-        while (state != ParallelEoSStreamProcessor.State.closed) {
-            log.trace("Loop: Poll broker");
-            ConsumerRecords<K, V> polledRecords = pollBrokerForRecords();
-
-            if (!polledRecords.isEmpty()) {
-                log.trace("Loop: Register work");
-                wm.registerWork(polledRecords);
+        committer.ifPresent(x -> x.claim());
+        try {
+            while (state != closed) {
+                log.trace("Loop: Broker poller: ({})", state);
+                if (state == running) {
+                    ConsumerRecords<K, V> polledRecords = pollBrokerForRecords();
 
-                // notify control work has been registered
-                pc.notifyNewWorkRegistered();
-            }
+                    if (!polledRecords.isEmpty()) {
+                        log.trace("Loop: Register work");
+                        wm.registerWork(polledRecords);
 
-            switch (state) {
-                case draining -> {
-                    doPause();
-                    // transition to closing
-                    state = ParallelEoSStreamProcessor.State.closing;
+                        // notify control work has been registered
+                        pc.notifyNewWorkRegistered();
+                    }
                 }
-                case closing -> {
-                    if (polledRecords.isEmpty()) {
+
+                maybeDoCommit();
+
+                switch (state) {
+                    case draining -> {
+                        doPause();
+                        transitionToCloseMaybe();
+                    }
+                    case closing -> {
                         doClose();
-                    } else {
-                        log.info("Subscriptions are paused, but records are still being drained (count: {})", polledRecords.count());
                     }
                 }
             }
+            log.debug("Broker poll thread finished, returning true to future");
+            return true;
+        } catch (Exception e) {
+            log.error("Unknown error", e);
+            throw e;
+        }
+    }
+
+    private void transitionToCloseMaybe() {
+        // make sure everything is committed
+        if (isResponsibleForCommits() && !wm.isRecordsAwaitingToBeCommitted()) {
+            // transition to closing
+            state = ParallelEoSStreamProcessor.State.closing;
         }
-        log.trace("Broker poll thread returning true");
-        return true;
     }
 
     private void doClose() {
-        log.debug("Closing {}, first closing consumer...", this.getClass().getSimpleName());
-        this.consumer.close(DrainingCloseable.DEFAULT_TIMEOUT);
-        log.debug("Consumer closed.");
-        state = ParallelEoSStreamProcessor.State.closed;
+        doPause();
+        maybeCloseConsumer();
+        state = closed;
+    }
+
+    /**
+     * To keep things simple, make sure the correct thread which can make a commit, is the one to close the consumer.
+     * This way, if partitions are revoked, the commit can be made inline.
+     */
+    private void maybeCloseConsumer() {
+        if (isResponsibleForCommits()) {
+            log.debug("Closing {}, first closing consumer...", this.getClass().getSimpleName());
+            this.consumerManager.close(DrainingCloseable.DEFAULT_TIMEOUT);
+            log.debug("Consumer closed.");
+        }
+    }
+
+    private boolean isResponsibleForCommits() {
+        return committer.isPresent();
     }
 
     private ConsumerRecords<K, V> pollBrokerForRecords() {
@@ -108,16 +165,8 @@ private ConsumerRecords<K, V> pollBrokerForRecords() {
 
         Duration thisLongPollTimeout = (state == ParallelEoSStreamProcessor.State.running) ? BrokerPollSystem.longPollTimeout : Duration.ofMillis(1); // Can't use Duration.ZERO - this causes Object#wait to wait forever
 
-        log.debug("Long polling broker with timeout {} seconds, might appear to sleep here if no data available on broker.", toSeconds(thisLongPollTimeout)); // java 8
-        ConsumerRecords<K, V> records;
-        try {
-            records = this.consumer.poll(thisLongPollTimeout);
-            log.debug("Poll completed normally and returned {}...", records.count());
-        } catch (WakeupException w) {
-            log.warn("Awoken from poll. State? {}", state);
-            records = new ConsumerRecords<>(UniMaps.of());
-        }
-        return records;
+        log.debug("Long polling broker with timeout {} seconds, might appear to sleep here if subs are paused, or no data available on broker.", toSeconds(thisLongPollTimeout)); // java 8
+        return consumerManager.poll(thisLongPollTimeout);
     }
 
     /**
@@ -126,9 +175,9 @@ private ConsumerRecords<K, V> pollBrokerForRecords() {
     public void drain() {
         // idempotent
         if (state != ParallelEoSStreamProcessor.State.draining) {
-            log.debug("Poll system signaling to drain...");
+            log.debug("Signaling poll system to drain, waking up consumer...");
             state = ParallelEoSStreamProcessor.State.draining;
-            consumer.wakeup();
+            consumerManager.wakeup();
         }
     }
 
@@ -139,8 +188,8 @@ private void doPause() {
         } else {
             paused = true;
             log.debug("Pausing subs");
-            Set<TopicPartition> assignment = consumer.assignment();
-            consumer.pause(assignment);
+            Set<TopicPartition> assignment = consumerManager.assignment();
+            consumerManager.pause(assignment);
         }
     }
 
@@ -151,7 +200,7 @@ public void closeAndWait() throws TimeoutException, ExecutionException {
             log.debug("Wait for loop to finish ending...");
             Future<Boolean> pollControlResult = pollControlThreadFuture.get();
             boolean interrupted = true;
-            while(interrupted) {
+            while (interrupted) {
                 try {
                     Boolean pollShutdownSuccess = pollControlResult.get(DrainingCloseable.DEFAULT_TIMEOUT.toMillis(), MILLISECONDS);
                     interrupted = false;
@@ -159,9 +208,9 @@ public void closeAndWait() throws TimeoutException, ExecutionException {
                         log.warn("Broker poll control thread not closed cleanly.");
                     }
                 } catch (InterruptedException e) {
-                    log.debug("Interrupted", e);
+                    log.debug("Interrupted waiting for broker poller thread to finish", e);
                 } catch (ExecutionException | TimeoutException e) {
-                    log.error("Execution or timeout exception", e);
+                    log.error("Execution or timeout exception waiting for broker poller thread to finish", e);
                     throw e;
                 }
             }
@@ -170,8 +219,9 @@ public void closeAndWait() throws TimeoutException, ExecutionException {
     }
 
     private void transitionToClosing() {
+        log.debug("Poller transitioning to closing, waking up consumer");
         state = ParallelEoSStreamProcessor.State.closing;
-        consumer.wakeup();
+        consumerManager.wakeup();
     }
 
     /**
@@ -192,11 +242,11 @@ private void managePauseOfSubscription() {
     private void resumeIfPaused() {
         // idempotent
         if (paused) {
-            log.debug("Resuming consumer");
-            Set<TopicPartition> pausedTopics = consumer.paused();
-            consumer.resume(pausedTopics);
+            log.debug("Resuming consumer, waking up");
+            Set<TopicPartition> pausedTopics = consumerManager.paused();
+            consumerManager.resume(pausedTopics);
             // trigger consumer to perform a new poll without the assignments paused, otherwise it will continue to long poll on nothing
-            consumer.wakeup();
+            consumerManager.wakeup();
             paused = false;
         }
     }
@@ -204,4 +254,34 @@ private void resumeIfPaused() {
     private boolean shouldThrottle() {
         return wm.shouldThrottle();
     }
+
+    /**
+     * Optionally blocks. Threadsafe
+     *
+     * @see CommitMode
+     */
+    @SneakyThrows
+    @Override
+    public void retrieveOffsetsAndCommit() {
+        // {@link Optional#ifPresentOrElse} only @since 9
+        ConsumerOffsetCommitter<K, V> comitter = committer.orElseThrow(() -> {
+            // shouldn't be here
+            throw new IllegalStateException("No committer configured");
+        });
+        comitter.commit();
+    }
+
+    /**
+     * Will silently skip if not configured with a committer
+     */
+    private void maybeDoCommit() {
+        committer.ifPresent(ConsumerOffsetCommitter::maybeDoCommit);
+    }
+
+    /**
+     * Wakeup if colling the broker
+     */
+    public void wakeup() {
+        consumerManager.wakeup();
+    }
 }
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ConsumerManager.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ConsumerManager.java
new file mode 100644
index 000000000..e96663d8b
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ConsumerManager.java
@@ -0,0 +1,114 @@
+package io.confluent.parallelconsumer;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.*;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.WakeupException;
+import pl.tlinkowski.unij.api.UniMaps;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Delegate for {@link KafkaConsumer}
+ */
+@Slf4j
+@RequiredArgsConstructor
+public class ConsumerManager<K, V> {
+
+    private final Consumer<K, V> consumer;
+
+    private final AtomicBoolean pollingBroker = new AtomicBoolean(false);
+
+    private int erroneousWakups = 0;
+    private int correctPollWakeups = 0;
+    private int noWakeups = 0;
+
+    ConsumerRecords<K, V> poll(Duration thisLongPollTimeout) {
+        ConsumerRecords<K, V> records;
+        try {
+            pollingBroker.set(true);
+            records = consumer.poll(thisLongPollTimeout);
+            log.debug("Poll completed normally and returned {}...", records.count());
+        } catch (WakeupException w) {
+            correctPollWakeups++;
+            log.debug("Awoken from broker poll", w);
+            records = new ConsumerRecords<>(UniMaps.of());
+        } finally {
+            pollingBroker.set(false);
+        }
+        return records;
+    }
+
+    /**
+     * Wakes up the consumer, but only if it's polling.
+     * <p>
+     * Otherwise we can interrupt other operations like {@link KafkaConsumer#commitSync()}.
+     */
+    public void wakeup() {
+        // boolean reduces the chances of a mis-timed call to wakeup, but doesn't prevent all spurious wake up calls to other methods like #commit
+        // if the call to wakeup happens /after/ the check for a wake up state inside #poll, then the next call will through the wake up exception (i.e. #commit)
+        if (pollingBroker.get()) {
+            log.debug("Waking up consumer");
+            consumer.wakeup();
+        }
+    }
+
+    public void commitSync(final Map<TopicPartition, OffsetAndMetadata> offsetsToSend) {
+        // we dont' want to be woken up during a commit, only polls
+        boolean inProgress = true;
+        noWakeups++;
+        while (inProgress) {
+            try {
+                consumer.commitSync(offsetsToSend);
+                inProgress = false;
+            } catch (WakeupException w) {
+                log.debug("Got woken up, retry. errors: " + erroneousWakups + " none: " + noWakeups + " correct:" + correctPollWakeups, w);
+                erroneousWakups++;
+            }
+        }
+    }
+
+    public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
+        // we dont' want to be woken up during a commit, only polls
+        boolean inProgress = true;
+        noWakeups++;
+        while (inProgress) {
+            try {
+                consumer.commitAsync(offsets, callback);
+                inProgress = false;
+            } catch (WakeupException w) {
+                log.debug("Got woken up, retry. errors: " + erroneousWakups + " none: " + noWakeups + " correct:" + correctPollWakeups, w);
+                erroneousWakups++;
+            }
+        }
+    }
+
+    public ConsumerGroupMetadata groupMetadata() {
+        return consumer.groupMetadata();
+    }
+
+    public void close(final Duration defaultTimeout) {
+        consumer.close(defaultTimeout);
+    }
+
+    public Set<TopicPartition> assignment() {
+        return consumer.assignment();
+    }
+
+    public void pause(final Set<TopicPartition> assignment) {
+        consumer.pause(assignment);
+    }
+
+    public Set<TopicPartition> paused() {
+        return consumer.paused();
+    }
+
+    public void resume(final Set<TopicPartition> pausedTopics) {
+        consumer.resume(pausedTopics);
+    }
+
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ConsumerOffsetCommitter.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ConsumerOffsetCommitter.java
new file mode 100644
index 000000000..bbe525851
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ConsumerOffsetCommitter.java
@@ -0,0 +1,203 @@
+package io.confluent.parallelconsumer;
+
+import io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.CONSUMER_SYNC;
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.TRANSACTIONAL_PRODUCER;
+
+/**
+ * Committer that uses the Kafka Consumer to commit either synchronously or asynchronously
+ *
+ * @see CommitMode
+ */
+@Slf4j
+public class ConsumerOffsetCommitter<K, V> extends AbstractOffsetCommitter<K, V> implements OffsetCommitter {
+
+    private final CommitMode commitMode;
+
+    /**
+     * Used to synchronise threads on changing {@link #commitCount}, {@link #commitPerformed} and constructing the
+     * {@link Condition} to wait on, that's all.
+     */
+    private final ReentrantLock commitLock = new ReentrantLock(true);
+
+    /**
+     * Used to signal to waiting threads, that their commit has been performed when requested.
+     *
+     * @see #commitPerformed
+     * @see #commitAndWaitForCondition()
+     */
+    private Condition commitPerformed = commitLock.newCondition();
+
+    /**
+     * The number of commits made. Use as a logical clock to sanity check expectations and synchronisation when commits
+     * are requested versus performed.
+     */
+    private final AtomicLong commitCount = new AtomicLong(0);
+
+    /**
+     * Set to true when a thread requests a commit to be performed, by the controling thread.
+     */
+    private final AtomicBoolean commitRequested = new AtomicBoolean(false);
+
+    private Optional<Thread> owningThread = Optional.empty();
+
+    public ConsumerOffsetCommitter(final ConsumerManager<K, V> newConsumer, final WorkManager<K, V> newWorkManager, final ParallelConsumerOptions options) {
+        super(newConsumer, newWorkManager);
+        commitMode = options.getCommitMode();
+        if (commitMode.equals(TRANSACTIONAL_PRODUCER)) {
+            throw new IllegalArgumentException("Cannot use " + commitMode + " when using " + this.getClass().getSimpleName());
+        }
+    }
+
+    // todo abstraction leak - find another way
+    private boolean direct = false;
+
+    /**
+     * Might block if using {@link CommitMode#CONSUMER_SYNC}
+     *
+     * @see CommitMode
+     */
+    void commit() {
+        if (isOwner()) {
+            // if owning thread is asking, then perform the commit directly (this is the thread that controls the consumer)
+            // this can happen when the system is closing, using Consumer commit mode, and partitions are revoked and we want to commit
+            direct = true;
+            retrieveOffsetsAndCommit();
+        } else if (isSync()) {
+            log.debug("Sync commit");
+            commitAndWaitForCondition();
+            log.debug("Finished waiting");
+        } else {
+            // async
+            // we just request the commit and hope
+            log.debug("Async commit to be requested");
+            requestCommitInternal();
+        }
+    }
+
+    @Override
+    protected void commitOffsets(final Map<TopicPartition, OffsetAndMetadata> offsetsToSend, final ConsumerGroupMetadata groupMetadata) {
+        if (offsetsToSend.isEmpty()) {
+            log.trace("Nothing to commit");
+            return;
+        }
+        switch (commitMode) {
+            case CONSUMER_SYNC -> {
+                log.debug("Committing offsets Sync");
+                consumerMgr.commitSync(offsetsToSend);
+            }
+            case CONSUMER_ASYNCHRONOUS -> {
+                //
+                log.debug("Committing offsets Async");
+                consumerMgr.commitAsync(offsetsToSend, (offsets, exception) -> {
+                    if (exception != null) {
+                        log.error("Error committing offsets", exception);
+                        // todo keep work in limbo until async response is received?
+                    }
+                });
+            }
+            default -> {
+                throw new IllegalArgumentException("Cannot use " + commitMode + " when using " + this.getClass().getSimpleName());
+            }
+        }
+    }
+
+    /**
+     * @see #commit()
+     */
+    @Override
+    protected void postCommit() {
+        // only signal if we are in sync mode, and current thread isn't the owner (if we are owner, then we are committing directly)
+        if (!direct && commitMode.equals(CONSUMER_SYNC))
+            signalCommitPerformed();
+    }
+
+    private boolean isOwner() {
+        return Thread.currentThread().equals(owningThread.orElse(null));
+    }
+
+    private void signalCommitPerformed() {
+        log.debug("Starting Signaling commit finished");
+        if (!commitLock.isHeldByCurrentThread())
+            throw new IllegalStateException("Lock already held");
+        commitLock.lock();
+        try {
+            commitCount.incrementAndGet();
+            log.debug("Signaling commit finished");
+            commitPerformed.signalAll();
+            log.debug("Finished Signaling commit finished");
+        } finally {
+            commitLock.unlock();
+        }
+    }
+
+    private void commitAndWaitForCondition() {
+        commitLock.lock();
+
+        try {
+            this.commitPerformed = commitLock.newCondition();
+            long currentCount = commitCount.get();
+            requestCommitInternal();
+            consumerMgr.wakeup();
+            while (currentCount == commitCount.get()) {
+                if (currentCount == commitCount.get()) {
+                    log.debug("Requesting commit again");
+                    requestCommitInternal();
+                } else {
+                    commitRequested.set(false);
+                }
+                try {
+                    log.debug("Waiting on commit");
+                    commitPerformed.await();
+                } catch (InterruptedException e) {
+                    log.debug("Interrupted waiting for commit condition", e);
+                }
+            }
+            log.debug("Signaled");
+        } finally {
+            commitLock.unlock();
+        }
+    }
+
+    private void requestCommitInternal() {
+        commitLock.lock();
+        try {
+            commitRequested.set(true);
+            consumerMgr.wakeup();
+        } finally {
+            commitLock.unlock();
+        }
+
+    }
+
+    void maybeDoCommit() {
+        commitLock.lock();
+        try {
+            if (commitRequested.get()) {
+                retrieveOffsetsAndCommit();
+            }
+        } finally {
+            commitLock.unlock();
+        }
+    }
+
+    public boolean isSync() {
+        return commitMode.equals(CONSUMER_SYNC);
+    }
+
+    public void claim() {
+        owningThread = Optional.of(Thread.currentThread());
+    }
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ErrorInUserFunctionException.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ErrorInUserFunctionException.java
new file mode 100644
index 000000000..d41b51594
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ErrorInUserFunctionException.java
@@ -0,0 +1,10 @@
+package io.confluent.parallelconsumer;
+
+/**
+ * This exception is only used when there is an exception thrown from code provided by the user.
+ */
+public class ErrorInUserFunctionException extends RuntimeException {
+    public ErrorInUserFunctionException(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/InternalError.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/InternalError.java
new file mode 100644
index 000000000..71c15870d
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/InternalError.java
@@ -0,0 +1,20 @@
+package io.confluent.parallelconsumer;
+
+public class InternalError extends RuntimeException {
+
+    public InternalError(final String message) {
+        super(message);
+    }
+
+    public InternalError(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+
+    public InternalError(final Throwable cause) {
+        super(cause);
+    }
+
+    public InternalError(final String message, final Throwable cause, final boolean enableSuppression, final boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/OffsetCommitter.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/OffsetCommitter.java
new file mode 100644
index 000000000..19d40baa0
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/OffsetCommitter.java
@@ -0,0 +1,5 @@
+package io.confluent.parallelconsumer;
+
+public interface OffsetCommitter {
+    void retrieveOffsetsAndCommit();
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumerOptions.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumerOptions.java
index a1ea68039..2654020b6 100644
--- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumerOptions.java
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumerOptions.java
@@ -4,16 +4,27 @@
  * Copyright (C) 2020 Confluent, Inc.
  */
 
+import io.confluent.csid.utils.StringUtils;
 import lombok.Builder;
 import lombok.Getter;
+import lombok.ToString;
+
+import java.util.Properties;
+
+import static io.confluent.csid.utils.StringUtils.msg;
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.TRANSACTIONAL_PRODUCER;
 
 /**
  * The options for the {@link ParallelEoSStreamProcessor} system.
  */
 @Getter
-@Builder
+@Builder(toBuilder = true)
+@ToString
 public class ParallelConsumerOptions {
 
+    /**
+     * The ordering guarantee to use.
+     */
     public enum ProcessingOrder {
         /**
          * No ordering is guaranteed, not even partition order. Fastest. Concurrency is at most the max number of
@@ -21,8 +32,8 @@ public enum ProcessingOrder {
          */
         UNORDERED,
         /**
-         * Process messages within a partition in order, but process multiple partitions in parallel. Similar to
-         * running more consumer for a topic. Concurrency is at most the number of partitions.
+         * Process messages within a partition in order, but process multiple partitions in parallel. Similar to running
+         * more consumer for a topic. Concurrency is at most the number of partitions.
          */
         PARTITION,
         /**
@@ -32,27 +43,88 @@ public enum ProcessingOrder {
         KEY
     }
 
+    /**
+     * The type of commit to be made, with either a transactions configured Producer where messages produced are
+     * committed back to the Broker along with the offsets they originated from, or with the faster simpler Consumer
+     * offset system either synchronously or asynchronously
+     */
+    public enum CommitMode {
+        /**
+         * Commits through the Producer using transactions. Slowest fot he options, but no duplicates in Kafka
+         * guaranteed (message replay may cause duplicates in external systems which is unavoidable with Kafka).
+         */
+        TRANSACTIONAL_PRODUCER,
+        /**
+         * Synchronous commits with the Consumer. Much faster than {@link #TRANSACTIONAL_PRODUCER}. Slower but
+         * potentially less duplicates than {@link #CONSUMER_ASYNCHRONOUS} upon replay.
+         */
+        CONSUMER_SYNC,
+        /**
+         * Fastest option, under normal conditions will have few of no duplicates. Under failure revocery may have more
+         * duplicates than {@link #CONSUMER_SYNC}.
+         */
+        CONSUMER_ASYNCHRONOUS
+    }
+
     /**
      * The order type to use
      */
     @Builder.Default
     private final ProcessingOrder ordering = ProcessingOrder.UNORDERED;
 
+    @Builder.Default
+    private final CommitMode commitMode = CommitMode.CONSUMER_ASYNCHRONOUS;
+
+    /**
+     * When using a produce flow, when producing the message, either
+     * <p>
+     * This is separate from using an IDEMPOTENT Producer, which can be used, along with {@link
+     * CommitMode#CONSUMER_SYNC} or {@link CommitMode#CONSUMER_ASYNCHRONOUS}.
+     * <p>
+     * Must be set to true if being used with a transactional producer. Producers state. However, forcing it to be
+     * specified makes the choice more verbose?
+     * <p>
+     * TODO we could just auto detect this from the
+     * <p>
+     * TODO delete in favor of CommitMode
+     */
+    @Builder.Default
+    private final boolean usingTransactionalProducer = false;
+
     /**
      * Don't have more than this many uncommitted messages in process
-     * TODO change this to per topic? global?
+     * <p>
+     * TODO docs - needs renaming. Messages aren't "uncommitted", they're just in the comitted offset map instead of the
+     * committed base offset
      */
     @Builder.Default
-    private final int maxUncommittedMessagesToHandlePerPartition = 1000;
+    private final int maxUncommittedMessagesToHandle = 1000;
 
     /**
      * Don't process any more than this many messages concurrently
+     * <p>
+     * TODO docs differentiate from thread count, vertx etc. remove to vertx module?
      */
     @Builder.Default
     private final int maxConcurrency = 100;
 
+    /**
+     * TODO docs. rename to max concurrency. differentiate between this and vertx threads
+     */
     @Builder.Default
     private final int numberOfThreads = 16;
 
+    // TODO remove - or include when checking passed property arguments instead of instances
+    //    using reflection instead
+//    @Builder.Default
+//    private final Properties producerConfig = new Properties();
 
+    public void validate() {
+        boolean commitModeIsTx = commitMode.equals(TRANSACTIONAL_PRODUCER);
+        if (this.isUsingTransactionalProducer() ^ commitModeIsTx) {
+            throw new IllegalArgumentException(msg("Using transaction producer mode ({}) without matching commit mode ({})",
+                    this.isUsingTransactionalProducer(),
+                    commitMode));
+        }
+    }
 }
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessor.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessor.java
index 98ffa39e6..e378f3530 100644
--- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessor.java
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessor.java
@@ -10,7 +10,6 @@
 import org.apache.kafka.clients.consumer.*;
 import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator;
 import org.apache.kafka.clients.producer.*;
-import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.utils.Time;
 import org.slf4j.MDC;
@@ -32,6 +31,7 @@
 import static io.confluent.csid.utils.BackportUtils.toSeconds;
 import static io.confluent.csid.utils.Range.range;
 import static io.confluent.csid.utils.StringUtils.msg;
+import static io.confluent.parallelconsumer.UserFunctions.carefullyRun;
 import static java.time.Duration.ofSeconds;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
@@ -42,6 +42,8 @@
 @Slf4j
 public class ParallelEoSStreamProcessor<K, V> implements ParallelStreamProcessor<K, V>, ConsumerRebalanceListener, Closeable {
 
+    private final ParallelConsumerOptions options;
+
     /**
      * Injectable clock for testing
      */
@@ -54,9 +56,8 @@ public class ParallelEoSStreamProcessor<K, V> implements ParallelStreamProcessor
 
     private Instant lastCommit = Instant.now();
 
-    private boolean inTransaction = false;
+    final ProducerManager<K, V> producerManager;
 
-    private final org.apache.kafka.clients.producer.Producer<K, V> producer;
     private final org.apache.kafka.clients.consumer.Consumer<K, V> consumer;
 
     /**
@@ -95,6 +96,8 @@ public class ParallelEoSStreamProcessor<K, V> implements ParallelStreamProcessor
      */
     private final AtomicBoolean currentlyPollingWorkCompleteMailBox = new AtomicBoolean();
 
+    private final OffsetCommitter committer;
+
     /**
      * The run state of the controller.
      *
@@ -122,47 +125,50 @@ enum State {
      *
      * @see ParallelConsumerOptions
      */
-    public ParallelEoSStreamProcessor(org.apache.kafka.clients.consumer.Consumer<K, V> consumer,
-                                      org.apache.kafka.clients.producer.Producer<K, V> producer,
-                                      ParallelConsumerOptions options) {
-        log.debug("Confluent async consumer initialise");
+    public ParallelEoSStreamProcessor(org.apache.kafka.clients.consumer.Consumer<K, V> newConsumer,
+                                      org.apache.kafka.clients.producer.Producer<K, V> newProducer,
+                                      ParallelConsumerOptions newOptions) {
+        log.info("Confluent Parallel Consumer initialise... Options: {}", newOptions);
 
-        Objects.requireNonNull(consumer);
-        Objects.requireNonNull(producer);
-        Objects.requireNonNull(options);
+        // todo delete?
+        ParallelEoSStreamProcessor.me = this;
 
-        checkNotSubscribed(consumer);
-        checkAutoCommitIsDisabled(consumer);
+        Objects.requireNonNull(newConsumer);
+        Objects.requireNonNull(newProducer);
+        Objects.requireNonNull(newOptions);
 
-        //
-        this.producer = producer;
-        this.consumer = consumer;
+        options = newOptions;
+        options.validate();
 
-        workerPool = Executors.newFixedThreadPool(options.getNumberOfThreads());
+        checkNotSubscribed(newConsumer);
+        checkAutoCommitIsDisabled(newConsumer);
 
-        //
-        this.wm = new WorkManager<>(options, consumer);
+        this.consumer = newConsumer;
 
-        //
-        this.brokerPollSubsystem = new BrokerPollSystem<>(consumer, wm, this);
+        this.workerPool = Executors.newFixedThreadPool(newOptions.getNumberOfThreads());
 
-        //
-        try {
-            log.debug("Initialising producer transaction session...");
-            producer.initTransactions();
-        } catch (KafkaException e) {
-            log.error("Make sure your producer is setup for transactions - specifically make sure it's {} is set.", ProducerConfig.TRANSACTIONAL_ID_CONFIG, e);
-            throw e;
+        this.wm = new WorkManager<>(newOptions, newConsumer);
+
+        ConsumerManager<K, V> consumerMgr = new ConsumerManager<>(newConsumer);
+        this.producerManager = new ProducerManager<>(newProducer, consumerMgr, this.wm, options);
+
+        this.brokerPollSubsystem = new BrokerPollSystem<>(consumerMgr, wm, this, newOptions);
+
+        if (options.isUsingTransactionalProducer()) {
+            this.committer = this.producerManager;
+        } else {
+            this.committer = this.brokerPollSubsystem;
         }
+
     }
 
-    private void checkNotSubscribed(org.apache.kafka.clients.consumer.Consumer<K, V> consumer) {
-        if (consumer instanceof MockConsumer)
+    private void checkNotSubscribed(org.apache.kafka.clients.consumer.Consumer<K, V> consumerToCheck) {
+        if (consumerToCheck instanceof MockConsumer)
             // disabled for unit tests which don't test rebalancing
             return;
-        Set<String> subscription = consumer.subscription();
-        Set<TopicPartition> assignment = consumer.assignment();
-        if (subscription.size() != 0 || assignment.size() != 0) {
+        Set<String> subscription = consumerToCheck.subscription();
+        Set<TopicPartition> assignment = consumerToCheck.assignment();
+        if (!subscription.isEmpty() || !assignment.isEmpty()) {
             throw new IllegalStateException("Consumer subscription must be managed by this class. Use " + this.getClass().getName() + "#subcribe methods instead.");
         }
     }
@@ -195,12 +201,19 @@ public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
 
     /**
      * Commit our offsets
+     * <p>
+     * Make sure the calling thread is the thread which performs commit - i.e. is the {@link OffsetCommitter}.
      */
     @Override
     public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
-        commitOffsetsThatAreReady();
-        wm.onPartitionsRevoked(partitions);
-        usersConsumerRebalanceListener.ifPresent(x -> x.onPartitionsRevoked(partitions));
+        try {
+            log.debug("Partitions revoked (onPartitionsRevoked), state: {}", state);
+            commitOffsetsThatAreReady();
+            wm.onPartitionsRevoked(partitions);
+            usersConsumerRebalanceListener.ifPresent(x -> x.onPartitionsRevoked(partitions));
+        } catch (Exception e) {
+            throw new InternalError("onPartitionsRevoked event error", e);
+        }
     }
 
     /**
@@ -255,7 +268,10 @@ private void checkAutoCommitIsDisabled(org.apache.kafka.clients.consumer.Consume
     public void poll(Consumer<ConsumerRecord<K, V>> usersVoidConsumptionFunction) {
         Function<ConsumerRecord<K, V>, List<Object>> wrappedUserFunc = (record) -> {
             log.trace("asyncPoll - Consumed a record ({}), executing void function...", record.offset());
-            usersVoidConsumptionFunction.accept(record);
+
+            carefullyRun(usersVoidConsumptionFunction, record);
+
+            log.trace("asyncPoll - user function finished ok.");
             return UniLists.of(); // user function returns no produce records, so we satisfy our api
         };
         Consumer<Object> voidCallBack = (ignore) -> log.trace("Void callback applied.");
@@ -268,15 +284,17 @@ public void pollAndProduce(Function<ConsumerRecord<K, V>, List<ProducerRecord<K,
                                Consumer<ConsumeProduceResult<K, V, K, V>> callback) {
         // wrap user func to add produce function
         Function<ConsumerRecord<K, V>, List<ConsumeProduceResult<K, V, K, V>>> wrappedUserFunc = (consumedRecord) -> {
-            List<ProducerRecord<K, V>> recordListToProduce = userFunction.apply(consumedRecord);
+
+            List<ProducerRecord<K, V>> recordListToProduce = carefullyRun(userFunction, consumedRecord);
+
             if (recordListToProduce.isEmpty()) {
                 log.warn("No result returned from function to send.");
             }
-            log.trace("asyncPoll and Stream - Consumed and a record ({}), and returning a derivative result record to be produced: {}", consumedRecord, recordListToProduce);
+            log.trace("asyncPoll and Stream - Consumed a record ({}), and returning a derivative result record to be produced: {}", consumedRecord, recordListToProduce);
 
             List<ConsumeProduceResult<K, V, K, V>> results = new ArrayList<>();
             for (ProducerRecord<K, V> toProduce : recordListToProduce) {
-                RecordMetadata produceResultMeta = produceMessage(toProduce);
+                RecordMetadata produceResultMeta = producerManager.produceMessage(toProduce);
                 var result = new ConsumeProduceResult<>(consumedRecord, toProduce, produceResultMeta);
                 results.add(result);
             }
@@ -286,30 +304,6 @@ public void pollAndProduce(Function<ConsumerRecord<K, V>, List<ProducerRecord<K,
         supervisorLoop(wrappedUserFunc, callback);
     }
 
-    /**
-     * Produce a message back to the broker.
-     * <p>
-     * Implementation uses the blocking API, performance upgrade in later versions, is not an issue for the common use
-     * case ({@link #poll(Consumer)}).
-     *
-     * @see #pollAndProduce(Function, Consumer)
-     */
-    RecordMetadata produceMessage(ProducerRecord<K, V> outMsg) {
-        // only needed if not using tx
-        Callback callback = (RecordMetadata metadata, Exception exception) -> {
-            if (exception != null) {
-                log.error("Error producing result message", exception);
-                throw new RuntimeException("Error producing result message", exception);
-            }
-        };
-        Future<RecordMetadata> send = producer.send(outMsg, callback);
-        try {
-            return send.get();
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
     @Override
     public void close() {
         // use a longer timeout, to cover for evey other step using the default
@@ -325,20 +319,22 @@ public void close(Duration timeout, DrainingMode drainMode) {
         } else {
             log.info("Signaling to close...");
 
-            switch (drainMode){
-                case DRAIN:
+            switch (drainMode) {
+                case DRAIN -> {
                     log.info("Will wait for all in flight to complete before");
                     transitionToDraining();
-                case DONT_DRAIN:
+                }
+                case DONT_DRAIN -> {
                     log.info("Not waiting for in flight to complete, will transition directly to closing");
                     transitionToClosing();
+                }
             }
 
             waitForClose(timeout);
         }
 
         if (controlThreadFuture.isPresent()) {
-            log.trace("Checking for control thread exception...");
+            log.debug("Checking for control thread exception...");
             Future<?> future = controlThreadFuture.get();
             future.get(toSeconds(timeout), SECONDS); // throws exception if supervisor saw one
         }
@@ -349,9 +345,10 @@ public void close(Duration timeout, DrainingMode drainMode) {
     private void waitForClose(Duration timeout) throws TimeoutException, ExecutionException {
         log.info("Waiting on closed state...");
         while (!state.equals(closed)) {
-            log.trace("Still waiting for system to close...");
             try {
-                boolean signaled = this.controlThreadFuture.get().get(toSeconds(timeout), SECONDS);
+                Future<Boolean> booleanFuture = this.controlThreadFuture.get();
+                log.debug("Blocking on control future");
+                boolean signaled = booleanFuture.get(toSeconds(timeout), SECONDS);
                 if (!signaled)
                     throw new TimeoutException("Timeout waiting for system to close (" + timeout + ")");
             } catch (InterruptedException e) {
@@ -361,6 +358,7 @@ private void waitForClose(Duration timeout) throws TimeoutException, ExecutionEx
                 log.error("Execution or timeout exception", e);
                 throw e;
             }
+            log.trace("Still waiting for system to close...");
         }
     }
 
@@ -371,12 +369,9 @@ private void doClose(Duration timeout) throws TimeoutException, ExecutionExcepti
         log.debug("Closing and waiting for broker poll system...");
         brokerPollSubsystem.closeAndWait();
 
-        log.debug("Closing producer, assuming no more in flight...");
-        if (this.inTransaction) {
-            // close started after tx began, but before work was done, otherwise a tx wouldn't have been started
-            producer.abortTransaction();
-        }
-        producer.close(timeout);
+        maybeCloseConsumer();
+
+        producerManager.close(timeout);
 
         log.debug("Shutting down execution pool...");
         List<Runnable> unfinished = workerPool.shutdownNow();
@@ -387,7 +382,7 @@ private void doClose(Duration timeout) throws TimeoutException, ExecutionExcepti
         log.trace("Awaiting worker pool termination...");
         boolean interrupted = true;
         while (interrupted) {
-            log.warn("Still interrupted");
+            log.debug("Still interrupted");
             try {
                 boolean terminationFinishedWithoutTimeout = workerPool.awaitTermination(toSeconds(DrainingCloseable.DEFAULT_TIMEOUT), SECONDS);
                 interrupted = false;
@@ -406,26 +401,43 @@ private void doClose(Duration timeout) throws TimeoutException, ExecutionExcepti
         this.state = closed;
     }
 
+    /**
+     * To keep things simple, make sure the correct thread which can make a commit, is the one to close the consumer.
+     * This way, if partitions are revoked, the commit can be made inline.
+     */
+    private void maybeCloseConsumer() {
+        if (isResponsibleForCommits()) {
+            consumer.close();
+        }
+    }
+
+    private boolean isResponsibleForCommits() {
+        return (committer instanceof ProducerManager);
+    }
+
     /**
      * Block the calling thread until no more messages are being processed.
      */
     @SneakyThrows
-    public void waitForNoInFlight(Duration timeout) {
-        log.debug("Waiting for no in flight...");
+    public void waitForProcessedNotCommitted(Duration timeout) {
+        log.debug("Waiting processed but not commmitted...");
         var timer = Time.SYSTEM.timer(timeout);
-        while (!noInFlight()) {
-            log.trace("Waiting for no in flight...");
+        while (wm.isRecordsAwaitingToBeCommitted()) {
+            log.trace("Waiting for no in processing...");
             Thread.sleep(100);
             timer.update();
             if (timer.isExpired()) {
-                throw new TimeoutException("Waiting for no more records in-flight");
+                throw new TimeoutException("Waiting for no more records in processing");
             }
         }
         log.debug("No longer anything in flight.");
     }
 
-    private boolean noInFlight() {
-        return !wm.isWorkRemaining() || areMyThreadsDone();
+    private boolean isRecordsAwaitingProcessing() {
+        boolean isRecordsAwaitingProcessing = wm.isRecordsAwaitingProcessing();
+        boolean threadsDone = areMyThreadsDone();
+        log.trace("isRecordsAwaitingProcessing {} || threadsDone {}", isRecordsAwaitingProcessing, threadsDone);
+        return isRecordsAwaitingProcessing || threadsDone;
     }
 
     private void transitionToDraining() {
@@ -462,6 +474,8 @@ private boolean areMyThreadsDone() {
      */
     protected <R> void supervisorLoop(Function<ConsumerRecord<K, V>, List<R>> userFunction,
                                       Consumer<R> callback) {
+        log.info("Control loop starting up...");
+
         if (state != State.unused) {
             throw new IllegalStateException(msg("Invalid state - the consumer cannot be used more than once (current " +
                     "state is {})", state));
@@ -469,26 +483,22 @@ protected <R> void supervisorLoop(Function<ConsumerRecord<K, V>, List<R>> userFu
             state = running;
         }
 
-        //
-        producer.beginTransaction();
-        this.inTransaction = true;
-
         // run main pool loop in thread
         Callable<Boolean> controlTask = () -> {
-            log.trace("Control task scheduled");
             Thread controlThread = Thread.currentThread();
             controlThread.setName("control");
+            log.trace("Control task scheduled");
             this.blockableControlThread = controlThread;
             while (state != closed) {
                 try {
                     controlLoop(userFunction, callback);
                 } catch (Exception e) {
-                    log.error("Error from poll control thread ({}), will attempt controlled shutdown, then rethrow...", e.getMessage(), e);
+                    log.error("Error from poll control thread, will attempt controlled shutdown, then rethrow. Error: " + e.getMessage(), e);
                     doClose(DrainingCloseable.DEFAULT_TIMEOUT); // attempt to close
                     throw new RuntimeException("Error from poll control thread: " + e.getMessage(), e);
                 }
             }
-            log.trace("Poll task ended (state:{}).", state);
+            log.info("Control loop ending clean (state:{})...", state);
             return true;
         };
 
@@ -503,8 +513,8 @@ protected <R> void supervisorLoop(Function<ConsumerRecord<K, V>, List<R>> userFu
      * Main control loop
      */
     private <R> void controlLoop(Function<ConsumerRecord<K, V>, List<R>> userFunction,
-                                 Consumer<R> callback) throws TimeoutException, ExecutionException {
-        if (state == running) {
+                                 Consumer<R> callback) throws TimeoutException, ExecutionException, InterruptedException {
+        if (state == running || state == draining) {
             log.trace("Loop: Get work");
             var records = wm.<R>maybeGetWork();
 
@@ -512,11 +522,21 @@ private <R> void controlLoop(Function<ConsumerRecord<K, V>, List<R>> userFunctio
             submitWorkToPool(userFunction, callback, records);
         }
 
+        if (state == running) {
+            if (!wm.isSufficientlyLoaded()) {
+                log.debug("Found not enough messages queued up, ensuring poller is awake");
+                brokerPollSubsystem.wakeup();
+            }
+        }
+
         log.trace("Loop: Process mailbox");
         processWorkCompleteMailBox();
 
-        log.trace("Loop: Maybe commit");
-        commitOffsetsMaybe();
+        if (state == running) {
+            // offsets will be committed when the consumer has its partitions revoked
+            log.trace("Loop: Maybe commit");
+            commitOffsetsMaybe();
+        }
 
         // run call back
         log.trace("Loop: Running {} loop end plugin(s)", controlLoopHooks.size());
@@ -524,10 +544,17 @@ private <R> void controlLoop(Function<ConsumerRecord<K, V>, List<R>> userFunctio
 
         log.debug("Current state: {}", state);
         switch (state) {
-            case draining -> drain();
-            case closing -> doClose(DrainingCloseable.DEFAULT_TIMEOUT);
+            case draining -> {
+                drain();
+            }
+            case closing -> {
+                doClose(DrainingCloseable.DEFAULT_TIMEOUT);
+            }
         }
 
+        // sanity - supervise the poller
+        brokerPollSubsystem.supervise();
+
         // end of loop
         log.trace("End of control loop, {} remaining in work manager. In state: {}", wm.getPartitionWorkRemainingCount(), state);
     }
@@ -535,7 +562,7 @@ private <R> void controlLoop(Function<ConsumerRecord<K, V>, List<R>> userFunctio
     private void drain() {
         log.debug("Signaling to drain...");
         brokerPollSubsystem.drain();
-        if (noInFlight()) {
+        if (!isRecordsAwaitingProcessing()) {
             transitionToClosing();
         }
     }
@@ -563,7 +590,7 @@ private void processWorkCompleteMailBox() {
         // blocking get the head of the queue
         WorkContainer<K, V> firstBlockingPoll = null;
         try {
-            log.debug("Blocking until next scheduled offset commit attempt for {}", timeout);
+            log.debug("Blocking poll on work until next scheduled offset commit attempt for {}", timeout);
             currentlyPollingWorkCompleteMailBox.getAndSet(true);
             // wait for work, with a timeout for sanity
             firstBlockingPoll = workMailBox.poll(timeout.toMillis(), MILLISECONDS);
@@ -587,7 +614,7 @@ private void processWorkCompleteMailBox() {
             try {
                 secondPollNonBlocking = workMailBox.poll(0, SECONDS);
             } catch (InterruptedException e) {
-                log.warn("Interrupted waiting on work results", e);
+                log.debug("Interrupted waiting on work results", e);
             }
             if (secondPollNonBlocking != null) {
                 results.add(secondPollNonBlocking);
@@ -610,7 +637,7 @@ private void commitOffsetsMaybe() {
         boolean commitFrequencyOK = toSeconds(elapsedSinceLast) >= toSeconds(timeBetweenCommits);
         if (commitFrequencyOK || lingeringOnCommitWouldBeBeneficial()) {
             if (!commitFrequencyOK) {
-                log.trace("Commit too frequent, but no benefit in lingering");
+                log.debug("Commit too frequent, but no benefit in lingering");
             }
             commitOffsetsThatAreReady();
             lastCommit = Instant.now();
@@ -643,7 +670,7 @@ private Duration getTimeToNextCommit() {
         if (state == running) {
             return getTimeBetweenCommits().minus(getTimeSinceLastCommit());
         } else {
-            log.debug("System not {}, so don't wait to commit", running);
+            log.debug("System not {} (state: {}), so don't wait to commit, only a small thread yield time", running, state);
             return Duration.ZERO;
         }
     }
@@ -653,61 +680,8 @@ private Duration getTimeSinceLastCommit() {
         return Duration.between(lastCommit, now);
     }
 
-    /**
-     * Get offsets from {@link WorkManager} that are ready to commit
-     */
     private void commitOffsetsThatAreReady() {
-        log.trace("Loop: Find completed work to commit offsets");
-        // todo shouldn't be removed until commit succeeds (there's no harm in committing the same offset twice)
-        Map<TopicPartition, OffsetAndMetadata> offsetsToSend = wm.findCompletedEligibleOffsetsAndRemove();
-        if (offsetsToSend.isEmpty()) {
-            log.trace("No offsets ready");
-        } else {
-            log.debug("Committing offsets for {} partition(s): {}", offsetsToSend.size(), offsetsToSend);
-            ConsumerGroupMetadata groupMetadata = consumer.groupMetadata();
-
-            producer.sendOffsetsToTransaction(offsetsToSend, groupMetadata);
-            // see {@link KafkaProducer#commit} this can be interrupted and is safe to retry
-            boolean notCommitted = true;
-            int retryCount = 0;
-            int arbitrarilyChosenLimitForArbitraryErrorSituation = 200;
-            Exception lastErrorSavedForRethrow = null;
-            while (notCommitted) {
-                if (retryCount > arbitrarilyChosenLimitForArbitraryErrorSituation) {
-                    String msg = msg("Retired too many times ({} > limit of {}), giving up. See error above.", retryCount, arbitrarilyChosenLimitForArbitraryErrorSituation);
-                    log.error(msg, lastErrorSavedForRethrow);
-                    throw new RuntimeException(msg, lastErrorSavedForRethrow);
-                }
-                try {
-                    if (producer instanceof MockProducer) {
-                        // see bug https://issues.apache.org/jira/browse/KAFKA-10382
-                        // KAFKA-10382 - MockProducer is not ThreadSafe, ideally it should be as the implementation it mocks is
-                        synchronized (producer) {
-                            producer.commitTransaction();
-                        }
-                    } else {
-                        producer.commitTransaction();
-                    }
-
-                    this.inTransaction = false;
-
-                    wm.onOffsetCommitSuccess(offsetsToSend);
-
-                    notCommitted = false;
-                    if (retryCount > 0) {
-                        log.warn("Commit success, but took {} tries.", retryCount);
-                    }
-                } catch (Exception e) {
-                    log.warn("Commit exception, will retry, have tried {} times (see KafkaProducer#commit)", retryCount, e);
-                    lastErrorSavedForRethrow = e;
-                    retryCount++;
-                }
-            }
-
-            // begin tx for next cycle
-            producer.beginTransaction();
-            this.inTransaction = true;
-        }
+        committer.retrieveOffsetsAndCommit();
     }
 
     protected void handleFutureResult(WorkContainer<K, V> wc) {
@@ -737,14 +711,16 @@ protected void onSuccess(WorkContainer<K, V> wc) {
     private <R> void submitWorkToPool(Function<ConsumerRecord<K, V>, List<R>> usersFunction,
                                       Consumer<R> callback,
                                       List<WorkContainer<K, V>> workToProcess) {
-
-        for (var work : workToProcess) {
-            // for each record, construct dispatch to the executor and capture a Future
-            log.trace("Sending work ({}) to pool", work);
-            Future outputRecordFuture = workerPool.submit(() -> {
-                return userFunctionRunner(usersFunction, callback, work);
-            });
-            work.setFuture(outputRecordFuture);
+        if (!workToProcess.isEmpty()) {
+            log.debug("New work incoming: {}, Pool stats: {}", workToProcess.size(), workerPool);
+            for (var work : workToProcess) {
+                // for each record, construct dispatch to the executor and capture a Future
+                log.trace("Sending work ({}) to pool", work);
+                Future outputRecordFuture = workerPool.submit(() -> {
+                    return userFunctionRunner(usersFunction, callback, work);
+                });
+                work.setFuture(outputRecordFuture);
+            }
         }
     }
 
@@ -777,7 +753,7 @@ protected <R> List<Tuple<ConsumerRecord<K, V>, R>> userFunctionRunner(Function<C
             return intermediateResults;
         } catch (Exception e) {
             // handle fail
-            log.debug("Error in user function", e);
+            log.debug("Error processing record", e);
             wc.onUserFunctionFailure();
             addToMailbox(wc); // always add on error
             throw e; // trow again to make the future failed
@@ -797,6 +773,7 @@ protected void onUserFunctionSuccess(WorkContainer<K, V> wc, List<?> resultsFrom
     protected void addToMailbox(WorkContainer<K, V> wc) {
         log.trace("Adding {} to mailbox...", wc);
         workMailBox.add(wc);
+        log.trace("Finished adding. {}", wc);
     }
 
     /**
@@ -809,8 +786,12 @@ protected void addToMailbox(WorkContainer<K, V> wc) {
      */
     void notifyNewWorkRegistered() {
         if (currentlyPollingWorkCompleteMailBox.get()) {
-            log.trace("Interrupting control thread: Knock knock, wake up! You've got mail (tm)!");
-            this.blockableControlThread.interrupt();
+            if (!producerManager.isTransactionInProgress()) {
+                log.trace("Interrupting control thread: Knock knock, wake up! You've got mail (tm)!");
+                this.blockableControlThread.interrupt();
+            } else {
+                log.trace("Would have interrupted control thread, but TX in progress");
+            }
         } else {
             log.trace("Work box not being polled currently, so thread not blocked, will come around to the bail box in the next looop.");
         }
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ProducerManager.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ProducerManager.java
new file mode 100644
index 000000000..746c40674
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ProducerManager.java
@@ -0,0 +1,311 @@
+package io.confluent.parallelconsumer;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.producer.*;
+import org.apache.kafka.clients.producer.internals.TransactionManager;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicPartition;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.time.Duration;
+import java.util.ConcurrentModificationException;
+import java.util.Map;
+import java.util.concurrent.Future;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static io.confluent.csid.utils.StringUtils.msg;
+import static io.confluent.parallelconsumer.ParallelEoSStreamProcessor.State.closing;
+
+@Slf4j
+public class ProducerManager<K, V> extends AbstractOffsetCommitter<K, V> implements OffsetCommitter {
+
+    protected final Producer<K, V> producer;
+
+    private final ParallelConsumerOptions options;
+
+    private final boolean producerIsConfiguredForTransactions;
+
+    /**
+     * The {@link KafkaProducer) isn't actually completely thread safe, at least when using it transactionally. We must
+     * be careful not to send messages to the producer, while we are committing a transaction - "Cannot call send in
+     * state COMMITTING_TRANSACTION".
+     */
+    private ReentrantReadWriteLock producerTransactionLock;
+
+    // nasty reflection
+    private Field txManagerField;
+    private Method txManagerMethodIsCompleting;
+    private Method txManagerMethodIsReady;
+
+    public ProducerManager(final Producer<K, V> newProducer, final ConsumerManager<K, V> newConsumer, final WorkManager<K, V> wm, ParallelConsumerOptions options) {
+        super(newConsumer, wm);
+        this.producer = newProducer;
+        this.options = options;
+
+        producerIsConfiguredForTransactions = setupReflection();
+
+        initProducer(newProducer);
+    }
+
+    private void initProducer(final Producer<K, V> newProducer) {
+        producerTransactionLock = new ReentrantReadWriteLock(true);
+
+        // String transactionIdProp = options.getProducerConfig().getProperty(ProducerConfig.TRANSACTIONAL_ID_CONFIG);
+        // boolean txIdSupplied = isBlank(transactionIdProp);
+        if (options.isUsingTransactionalProducer()) {
+            if (!producerIsConfiguredForTransactions) {
+                throw new IllegalArgumentException("Using transactional option, yet Producer doesn't have a transaction ID - Producer needs a transaction id");
+            }
+            try {
+                log.debug("Initialising producer transaction session...");
+                producer.initTransactions();
+                producer.beginTransaction();
+            } catch (KafkaException e) {
+                log.error("Make sure your producer is setup for transactions - specifically make sure it's {} is set.", ProducerConfig.TRANSACTIONAL_ID_CONFIG, e);
+                throw e;
+            }
+        } else {
+            if (producerIsConfiguredForTransactions) {
+                throw new IllegalArgumentException("Not using non-transactional option, but Producer has a transaction ID - Producer must not have a transaction ID for this option");
+            }
+        }
+    }
+
+    /**
+     * Nasty reflection but better than relying on user supplying their config
+     *
+     * @see ParallelEoSStreamProcessor#checkAutoCommitIsDisabled
+     */
+    @SneakyThrows
+    private boolean getProducerIsTransactional() {
+        if (producer instanceof MockProducer) {
+            // can act as both, delegate to user selection
+            return options.isUsingTransactionalProducer();
+        } else {
+            TransactionManager transactionManager = getTransactionManager();
+            if (transactionManager == null) {
+                return false;
+            } else {
+                return transactionManager.isTransactional();
+            }
+        }
+    }
+
+    @SneakyThrows
+    private TransactionManager getTransactionManager() {
+        if (txManagerField == null) return null;
+        TransactionManager transactionManager = (TransactionManager) txManagerField.get(producer);
+        return transactionManager;
+    }
+
+    /**
+     * Produce a message back to the broker.
+     * <p>
+     * Implementation uses the blocking API, performance upgrade in later versions, is not an issue for the more common
+     * use case where messages aren't produced.
+     *
+     * @see ParallelConsumer#poll
+     * @see ParallelStreamProcessor#pollAndProduce
+     */
+    RecordMetadata produceMessage(ProducerRecord<K, V> outMsg) {
+        // only needed if not using tx
+        Callback callback = (RecordMetadata metadata, Exception exception) -> {
+            if (exception != null) {
+                log.error("Error producing result message", exception);
+                throw new RuntimeException("Error producing result message", exception);
+            }
+        };
+
+        ReentrantReadWriteLock.ReadLock readLock = producerTransactionLock.readLock();
+        readLock.lock();
+        Future<RecordMetadata> send;
+        try {
+            send = producer.send(outMsg, callback);
+        } finally {
+            readLock.unlock();
+        }
+
+        // wait on the send results
+        try {
+            return send.get();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    protected void preAcquireWork() {
+        acquireCommitLock();
+    }
+
+    @Override
+    protected void postCommit() {
+        // only release lock when commit successful
+        if (producerTransactionLock.getWriteHoldCount() > 1) // sanity
+            throw new ConcurrentModificationException("Lock held too many times, won't be released problem and will cause deadlock");
+
+        releaseCommitLock();
+    }
+
+    @Override
+    protected void commitOffsets(final Map<TopicPartition, OffsetAndMetadata> offsetsToSend, final ConsumerGroupMetadata groupMetadata) {
+        log.debug("Transactional offset commit starting");
+        if (!options.isUsingTransactionalProducer()) {
+            throw new IllegalStateException("Bug: cannot use if not using transactional producer");
+        }
+
+        producer.sendOffsetsToTransaction(offsetsToSend, groupMetadata);
+        // see {@link KafkaProducer#commit} this can be interrupted and is safe to retry
+        boolean committed = false;
+        int retryCount = 0;
+        int arbitrarilyChosenLimitForArbitraryErrorSituation = 200;
+        Exception lastErrorSavedForRethrow = null;
+        while (!committed) {
+            if (retryCount > arbitrarilyChosenLimitForArbitraryErrorSituation) {
+                String msg = msg("Retired too many times ({} > limit of {}), giving up. See error above.", retryCount, arbitrarilyChosenLimitForArbitraryErrorSituation);
+                log.error(msg, lastErrorSavedForRethrow);
+                throw new RuntimeException(msg, lastErrorSavedForRethrow);
+            }
+            try {
+                if (producer instanceof MockProducer) {
+                    // see bug https://issues.apache.org/jira/browse/KAFKA-10382
+                    // KAFKA-10382 - MockProducer is not ThreadSafe, ideally it should be as the implementation it mocks is
+                    synchronized (producer) {
+                        producer.commitTransaction();
+                        producer.beginTransaction();
+                    }
+                } else {
+
+                    // producer commit lock should already be acquired at this point, before work was retrieved to commit,
+                    // so that more messages don't sneak into this tx block - the consumer records of which won't yet be
+                    // in this offset collection
+                    ensureLockHeld();
+
+                    boolean retrying = retryCount > 0;
+                    if (retrying) {
+                        if (isTransactionCompleting()) {
+                            // try wait again
+                            producer.commitTransaction();
+                        }
+                        if (isTransactionReady()) {
+                            // tx has completed since we last tried, start a new one
+                            producer.beginTransaction();
+                        }
+                        boolean ready = (lastErrorSavedForRethrow != null) ? !lastErrorSavedForRethrow.getMessage().contains("Invalid transition attempted from state READY to state COMMITTING_TRANSACTION") : true;
+                        if (ready) {
+                            // try again
+                            log.error("Was already ready - tx completed between interrupt and retry");
+                        }
+                    } else {
+                        // happy path
+                        producer.commitTransaction();
+                        producer.beginTransaction();
+                    }
+                }
+
+                committed = true;
+                if (retryCount > 0) {
+                    log.warn("Commit success, but took {} tries.", retryCount);
+                }
+            } catch (Exception e) {
+                log.warn("Commit exception, will retry, have tried {} times (see KafkaProducer#commit)", retryCount, e);
+                lastErrorSavedForRethrow = e;
+                retryCount++;
+            }
+        }
+    }
+
+    /**
+     * @return boolean which shows if we are setup for transactions or now
+     */
+    @SneakyThrows
+    private boolean setupReflection() {
+        if (producer instanceof KafkaProducer) {
+            txManagerField = producer.getClass().getDeclaredField("transactionManager");
+            txManagerField.setAccessible(true);
+
+            boolean producerIsConfiguredForTransactions = getProducerIsTransactional();
+            if (producerIsConfiguredForTransactions) {
+                TransactionManager transactionManager = getTransactionManager();
+                txManagerMethodIsCompleting = transactionManager.getClass().getDeclaredMethod("isCompleting");
+                txManagerMethodIsCompleting.setAccessible(true);
+
+                txManagerMethodIsReady = transactionManager.getClass().getDeclaredMethod("isReady");
+                txManagerMethodIsReady.setAccessible(true);
+            }
+            return producerIsConfiguredForTransactions;
+        } else if (producer instanceof MockProducer) {
+            // can act as both, delegate to user selection
+            return options.isUsingTransactionalProducer();
+        } else {
+            // unknown
+            return false;
+        }
+    }
+
+    /**
+     * TODO talk about alternatives to this brute force approach for retrying committing transactions
+     */
+    @SneakyThrows
+    private boolean isTransactionCompleting() {
+        if (producer instanceof MockProducer) return false;
+        return (boolean) txManagerMethodIsCompleting.invoke(getTransactionManager());
+    }
+
+    /**
+     * TODO talk about alternatives to this brute force approach for retrying committing transactions
+     */
+    @SneakyThrows
+    private boolean isTransactionReady() {
+        if (producer instanceof MockProducer) return true;
+        return (boolean) txManagerMethodIsReady.invoke(getTransactionManager());
+    }
+
+    /**
+     * Assumes the system is drained at this point, or draining is not desired.
+     */
+    public void close(final Duration timeout) {
+        log.debug("Closing producer, assuming no more in flight...");
+        if (options.isUsingTransactionalProducer() && !isTransactionReady()) {
+            acquireCommitLock();
+            try {
+                // close started after tx began, but before work was done, otherwise a tx wouldn't have been started
+                producer.abortTransaction();
+            } finally {
+                releaseCommitLock();
+            }
+        }
+        producer.close(timeout);
+    }
+
+    private void acquireCommitLock() {
+        if (producerTransactionLock.getWriteHoldCount() > 0)
+            throw new ConcurrentModificationException("Lock already held");
+        ReentrantReadWriteLock.WriteLock writeLock = producerTransactionLock.writeLock();
+        if (producerTransactionLock.isWriteLocked() && !producerTransactionLock.isWriteLockedByCurrentThread()) {
+            throw new ConcurrentModificationException(this.getClass().getSimpleName() + " is not safe for multi-threaded access");
+        }
+        writeLock.lock();
+    }
+
+    private void releaseCommitLock() {
+        log.trace("Release commit lock");
+        ReentrantReadWriteLock.WriteLock writeLock = producerTransactionLock.writeLock();
+        if (!producerTransactionLock.isWriteLockedByCurrentThread())
+            throw new IllegalStateException("Not held be me");
+        writeLock.unlock();
+    }
+
+    private void ensureLockHeld() {
+        if (!producerTransactionLock.isWriteLockedByCurrentThread())
+            throw new IllegalStateException("Expected commit lock to be held");
+    }
+
+    public boolean isTransactionInProgress() {
+        return producerTransactionLock.isWriteLocked();
+    }
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/UserFunctions.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/UserFunctions.java
new file mode 100644
index 000000000..45c26cb8b
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/UserFunctions.java
@@ -0,0 +1,38 @@
+package io.confluent.parallelconsumer;
+
+import lombok.experimental.UtilityClass;
+
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+@UtilityClass
+public class UserFunctions {
+
+    public static final String MSG = "Error occurred in code supplied by user";
+
+    public static <T, U, R> R carefullyRun(BiFunction<T, U, R> wrappedFunction, T t, U u) {
+        try {
+            return wrappedFunction.apply(t, u);
+        } catch (Exception e) {
+            throw new ErrorInUserFunctionException(MSG, e);
+        }
+    }
+
+    public static <A, B> B carefullyRun(Function<A, B> wrappedFunction, A a) {
+        try {
+            return wrappedFunction.apply(a);
+        } catch (Exception e) {
+            throw new ErrorInUserFunctionException(MSG, e);
+        }
+    }
+
+    public static <A> void carefullyRun(Consumer<A> wrappedFunction, A a) {
+        try {
+            wrappedFunction.accept(a);
+        } catch (Exception e) {
+            throw new ErrorInUserFunctionException(MSG, e);
+        }
+    }
+
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/WorkManager.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/WorkManager.java
index 0f360ddae..fe54543b1 100644
--- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/WorkManager.java
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/WorkManager.java
@@ -19,9 +19,11 @@
 import pl.tlinkowski.unij.api.UniLists;
 import pl.tlinkowski.unij.api.UniSets;
 
+import java.time.Duration;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
 import static io.confluent.csid.utils.KafkaUtils.toTP;
@@ -73,10 +75,13 @@ public class WorkManager<K, V> implements ConsumerRebalanceListener {
     private int inFlightCount = 0;
 
     /**
-     * The multiple of {@link ParallelConsumerOptions#getMaxConcurrency()} that should be pre-loaded awaiting processing.
-     * Consumer already pipelines, so we shouldn't need to pipeline ourselves too much.
+     * The multiple of {@link ParallelConsumerOptions#getMaxConcurrency()} that should be pre-loaded awaiting
+     * processing. Consumer already pipelines, so we shouldn't need to pipeline ourselves too much.
+     * <p>
+     * Note how this relates to {@link BrokerPollSystem#getLongPollTimeout()} - if longPollTimeout is high and loading
+     * factor is low, there may not be enough messages queued up to satisfy demand.
      */
-    private final int loadingFactor = 2;
+    private final int loadingFactor = 3;
 
     /**
      * Useful for testing
@@ -262,7 +267,7 @@ public <R> List<WorkContainer<K, V>> maybeGetWork() {
      * @param requestedMaxWorkToRetrieve ignored unless less than {@link ParallelConsumerOptions#maxConcurrency}
      */
     public List<WorkContainer<K, V>> maybeGetWork(int requestedMaxWorkToRetrieve) {
-        int minWorkToGetSetting = min(min(requestedMaxWorkToRetrieve, options.getMaxConcurrency()), options.getMaxUncommittedMessagesToHandlePerPartition());
+        int minWorkToGetSetting = min(min(requestedMaxWorkToRetrieve, options.getMaxConcurrency()), options.getMaxUncommittedMessagesToHandle());
         int workToGetDelta = minWorkToGetSetting - getInFlightCount();
 
         // optimise early
@@ -321,8 +326,9 @@ public List<WorkContainer<K, V>> maybeGetWork(int requestedMaxWorkToRetrieve) {
             work.addAll(shardWork);
         }
 
-        log.debug("Returning {} records of work", work.size());
+        log.debug("Got {} records of work", work.size());
         inFlightCount += work.size();
+
         return work;
     }
 
@@ -367,8 +373,14 @@ public int getShardWorkRemainingCount() {
         return count;
     }
 
-    boolean isWorkRemaining() {
-        return getPartitionWorkRemainingCount() > 0;
+    boolean isRecordsAwaitingProcessing() {
+        int partitionWorkRemainingCount = getShardWorkRemainingCount();
+        return partitionWorkRemainingCount > 0;
+    }
+
+    boolean isRecordsAwaitingToBeCommitted() {
+        int partitionWorkRemainingCount = getPartitionWorkRemainingCount();
+        return partitionWorkRemainingCount > 0;
     }
 
     public WorkContainer<K, V> getWorkContainerForRecord(ConsumerRecord<K, V> rec) {
@@ -388,7 +400,8 @@ boolean hasComittableOffsets() {
     }
 
     /**
-     * TODO: This entire loop could be possibly redundant, if we instead track low water mark, and incomplete offsets as work is submitted and returned.
+     * TODO: This entire loop could be possibly redundant, if we instead track low water mark, and incomplete offsets as
+     * work is submitted and returned.
      */
     @SneakyThrows
     <R> Map<TopicPartition, OffsetAndMetadata> findCompletedEligibleOffsetsAndRemove(boolean remove) {
@@ -480,7 +493,8 @@ <R> Map<TopicPartition, OffsetAndMetadata> findCompletedEligibleOffsetsAndRemove
     /**
      * Once all the offset maps have been calculated, check if they're too big, and if so, remove all of them.
      * <p>
-     * Implication of this is that if the system has to recover from this offset, then it will have to replay all the messages that were otherwise complete.
+     * Implication of this is that if the system has to recover from this offset, then it will have to replay all the
+     * messages that were otherwise complete.
      *
      * @see OffsetMapCodecManager#DefaultMaxMetadataSize
      */
@@ -491,7 +505,7 @@ private void maybeStripOffsetPayload(Map<TopicPartition, OffsetAndMetadata> offs
         //  retaining the offset map feature, at the cost of potential performance by hitting a soft maximum in our uncommitted concurrent processing.
         if (totalOffsetMetaSize > OffsetMapCodecManager.DefaultMaxMetadataSize) {
             log.warn("Offset map data too large (size: {}) to fit in metadata payload - stripping offset map out. " +
-                    "See kafka.coordinator.group.OffsetConfig#DefaultMaxMetadataSize = 4096",
+                            "See kafka.coordinator.group.OffsetConfig#DefaultMaxMetadataSize = 4096",
                     totalOffsetMetaSize);
             // strip payload
             for (var entry : offsetsToSend.entrySet()) {
@@ -507,7 +521,8 @@ private void maybeStripOffsetPayload(Map<TopicPartition, OffsetAndMetadata> offs
      * Truncate our tracked offsets as a commit was successful, so the low water mark rises, and we dont' need to track
      * as much anymore.
      * <p>
-     * When commits are made to broker, we can throw away all the individually tracked offsets before the committed offset.
+     * When commits are made to broker, we can throw away all the individually tracked offsets before the committed
+     * offset.
      */
     public void onOffsetCommitSuccess(Map<TopicPartition, OffsetAndMetadata> offsetsToSend) {
         // partitionOffsetHighWaterMarks this will get overwritten in due course
@@ -522,18 +537,18 @@ public void onOffsetCommitSuccess(Map<TopicPartition, OffsetAndMetadata> offsets
     }
 
     public boolean shouldThrottle() {
-        return isOverMax();
+        return isSufficientlyLoaded();
     }
 
-    private boolean isOverMax() {
+    boolean isSufficientlyLoaded() {
         int remaining = getPartitionWorkRemainingCount();
-        boolean loadedEnough = remaining > options.getMaxConcurrency() * loadingFactor;
-        boolean overMaxInFlight = remaining > options.getMaxUncommittedMessagesToHandlePerPartition();
-        boolean isOverMax = loadedEnough || overMaxInFlight;
-        if (isOverMax) {
-            log.debug("loadedEnough {} || overMaxInFlight {}", loadedEnough, overMaxInFlight);
+        boolean loadedEnoughInPipeline = remaining > options.getMaxConcurrency() * loadingFactor;
+        boolean overMaxUncommitted = remaining > options.getMaxUncommittedMessagesToHandle();
+        boolean remainingIsSufficient = loadedEnoughInPipeline || overMaxUncommitted;
+        if (remainingIsSufficient) {
+            log.debug("loadedEnoughInPipeline {} || overMaxUncommitted {}", loadedEnoughInPipeline, overMaxUncommitted);
         }
-        return isOverMax;
+        return remainingIsSufficient;
     }
 
     public int getInFlightCount() {
diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/CloseAndOpenOffsetTest.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/CloseAndOpenOffsetTest.java
index bb19351af..47c26403a 100644
--- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/CloseAndOpenOffsetTest.java
+++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/CloseAndOpenOffsetTest.java
@@ -4,6 +4,8 @@
  */
 package io.confluent.parallelconsumer.integrationTests;
 
+import io.confluent.csid.utils.KafkaTestUtils;
+import io.confluent.parallelconsumer.ParallelConsumer;
 import io.confluent.parallelconsumer.ParallelEoSStreamProcessor;
 import io.confluent.parallelconsumer.ParallelConsumerOptions;
 import io.confluent.parallelconsumer.OffsetMapCodecManager;
@@ -11,12 +13,11 @@
 import io.confluent.csid.utils.Range;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.*;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.TopicPartition;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
@@ -25,12 +26,16 @@
 import pl.tlinkowski.unij.api.UniSets;
 
 import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Set;
+import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ConcurrentSkipListSet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.TRANSACTIONAL_PRODUCER;
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder.UNORDERED;
 import static java.time.Duration.ofSeconds;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Awaitility.await;
@@ -43,7 +48,7 @@
 public class CloseAndOpenOffsetTest extends KafkaTest<String, String> {
 
     Duration normalTimeout = ofSeconds(5);
-    Duration debugTimeout = Duration.ofMinutes(10);
+    Duration debugTimeout = Duration.ofMinutes(1);
 
     // use debug timeout while debugging
 //    Duration timeoutToUse = debugTimeout;
@@ -56,6 +61,14 @@ void setup() {
         rebalanceTopic = "close-and-open-" + RandomUtils.nextInt();
     }
 
+    /**
+     * publish some messages some fail shutdown startup again consume again - check we only consume the failed messages
+     * <p>
+     * Sometimes fails as 5 is not comitted in the first run and comes out in the 2nd
+     * <p>
+     * NB: messages 4 and 2 are made to fail
+     */
+    @Timeout(value = 60)
     @SneakyThrows
     @Test
     void offsetsOpenClose() {
@@ -67,7 +80,11 @@ void offsetsOpenClose() {
         }
 
         // 1 client
-        ParallelConsumerOptions options = ParallelConsumerOptions.builder().ordering(ParallelConsumerOptions.ProcessingOrder.UNORDERED).build();
+        ParallelConsumerOptions options = ParallelConsumerOptions.builder()
+                .ordering(UNORDERED)
+                .usingTransactionalProducer(true)
+                .commitMode(TRANSACTIONAL_PRODUCER)
+                .build();
         kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "ONE-my-client");
         KafkaConsumer<String, String> newConsumerOne = kcu.createNewConsumer();
 
@@ -79,7 +96,7 @@ void offsetsOpenClose() {
         asyncOne.subscribe(UniLists.of(rebalanceTopic));
 
         // read some messages
-        var readByOne = new ArrayList<ConsumerRecord<String, String>>();
+        var readByOne = new ConcurrentLinkedQueue<ConsumerRecord<String, String>>();
         asyncOne.poll(x -> {
             log.info("Read by consumer ONE: {}", x);
             if (x.value().equals("4")) {
@@ -93,6 +110,9 @@ void offsetsOpenClose() {
             readByOne.add(x);
         });
 
+        // wait for initial 0 commit
+        Thread.sleep(500);
+
         //
         send(rebalanceTopic, 0, 0);
         send(rebalanceTopic, 0, 1);
@@ -102,12 +122,33 @@ void offsetsOpenClose() {
         send(rebalanceTopic, 0, 5);
 
         // all are processed except msg 2 and 4, which holds up the queue
-        await().atMost(debugTimeout).untilAsserted(() -> assertThat(readByOne).hasSize(4));
+        await().alias("check all except 2 and 4 are processed").atMost(normalTimeout).untilAsserted(() -> {
+                    ArrayList<ConsumerRecord<String, String>> copy = new ArrayList<>(readByOne);
+                    assertThat(copy.stream()
+                            .map(x -> x.value()).collect(Collectors.toList()))
+                            .containsOnly("0", "1", "3", "5");
+                }
+        );
+
+        // wait until all expected records have been processed and committed
+        // need to wait for final message processing's offset data to be committed
+        // TODO test for event/trigger instead - could consume offsets topic but have to decode the binary
+        // could listen to a produce topic, but currently it doesn't use the produce flow
+        // could add a commit listener to the api, but that's heavy just for this?
+        // could use Consumer#committed to check and decode, but it's not thread safe
+        // sleep is lazy but much much simpler
+        Thread.sleep(500);
 
         // commit what we've done so far, don't wait for failing messages to be retried (message 4)
         log.info("Closing consumer, committing offset map");
         asyncOne.closeDontDrainFirst();
 
+        await().alias("check all except 2 and 4 are processed").atMost(normalTimeout).untilAsserted(() ->
+                assertThat(readByOne.stream()
+                        .map(x -> x.value()).collect(Collectors.toList()))
+                        .containsOnly("0", "1", "3", "5"));
+
+
         //
         kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "THREE-my-client");
         KafkaConsumer<String, String> newConsumerThree = kcu.createNewConsumer();
@@ -116,14 +157,16 @@ void offsetsOpenClose() {
             asyncThree.subscribe(UniLists.of(rebalanceTopic));
 
             // read what we're given
-            var readByThree = new ArrayList<ConsumerRecord<String, String>>();
+            var readByThree = new ConcurrentLinkedQueue<ConsumerRecord<String, String>>();
             asyncThree.poll(x -> {
                 log.info("Read by consumer THREE: {}", x.value());
                 readByThree.add(x);
             });
 
             // only 2 and 4 should be delivered again, as everything else was processed successfully
-            await().atMost(timeoutToUse).untilAsserted(() -> assertThat(readByThree).extracting(ConsumerRecord::value).containsExactlyInAnyOrder("2", "4"));
+            await().atMost(timeoutToUse).untilAsserted(() ->
+                    assertThat(readByThree).extracting(ConsumerRecord::value)
+                            .containsExactlyInAnyOrder("2", "4"));
         }
     }
 
@@ -160,23 +203,33 @@ void correctOffsetVerySimple() {
 
         KafkaConsumer<String, String> consumer = kcu.createNewConsumer();
         KafkaProducer<String, String> producerOne = kcu.createNewProducer(true);
-        ParallelConsumerOptions options = ParallelConsumerOptions.builder().ordering(ParallelConsumerOptions.ProcessingOrder.UNORDERED).build();
+        ParallelConsumerOptions options = ParallelConsumerOptions.builder()
+                .ordering(UNORDERED)
+                .usingTransactionalProducer(true)
+                .commitMode(TRANSACTIONAL_PRODUCER)
+                .build();
 
         try (var asyncOne = new ParallelEoSStreamProcessor<>(consumer, producerOne, options)) {
 
             asyncOne.subscribe(UniLists.of(topic));
 
             var readByOne = new ArrayList<ConsumerRecord<String, String>>();
-            asyncOne.poll(readByOne::add);
+            asyncOne.poll(msg -> {
+                log.debug("Reading {}", msg);
+                readByOne.add(msg);
+            });
 
             // the single message is processed
             await().untilAsserted(() -> assertThat(readByOne)
                     .extracting(ConsumerRecord::value)
                     .containsExactly("0"));
 
+        } finally {
+            log.debug("asyncOne closed");
         }
 
         //
+        log.debug("Starting up new client");
         kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "THREE-my-client");
         KafkaConsumer<String, String> newConsumerThree = kcu.createNewConsumer();
         KafkaProducer<String, String> producerThree = kcu.createNewProducer(true);
@@ -220,7 +273,11 @@ void largeNumberOfMessagesSmallOffsetBitmap() {
 
         KafkaConsumer<String, String> consumer = kcu.createNewConsumer();
         KafkaProducer<String, String> producerOne = kcu.createNewProducer(true);
-        ParallelConsumerOptions options = ParallelConsumerOptions.builder().ordering(ParallelConsumerOptions.ProcessingOrder.UNORDERED).build();
+        ParallelConsumerOptions options = ParallelConsumerOptions.builder()
+                .ordering(UNORDERED)
+                .usingTransactionalProducer(true)
+                .commitMode(TRANSACTIONAL_PRODUCER)
+                .build();
         var asyncOne = new ParallelEoSStreamProcessor<>(consumer, producerOne, options);
 
         asyncOne.subscribe(UniLists.of(topic));
diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/KafkaTest.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/KafkaTest.java
index 43e7afb2e..14501362b 100644
--- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/KafkaTest.java
+++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/KafkaTest.java
@@ -6,7 +6,6 @@
 
 import io.confluent.parallelconsumer.integrationTests.utils.KafkaClientUtils;
 import io.confluent.csid.testcontainers.FilteredTestContainerSlf4jLogConsumer;
-import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.kafka.clients.admin.CreateTopicsResult;
 import org.apache.kafka.clients.admin.NewTopic;
@@ -18,11 +17,14 @@
 import org.testcontainers.utility.DockerImageName;
 import pl.tlinkowski.unij.api.UniLists;
 
+import java.util.concurrent.ExecutionException;
+
 import static org.apache.commons.lang3.RandomUtils.nextInt;
 import static org.assertj.core.api.Assertions.assertThat;
 
 @Testcontainers
 @Slf4j
+// TODO rename to broker integration test
 public abstract class KafkaTest<K, V> {
 
     int numPartitions = 2; // as default consumer settings are max request 50 max per partition 1 - 50/1=50
@@ -37,6 +39,8 @@ public abstract class KafkaTest<K, V> {
             .withEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1") //transaction.state.log.replication.factor
             .withEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1") //transaction.state.log.min.isr
             .withEnv("KAFKA_TRANSACTION_STATE_LOG_NUM_PARTITIONS", "1") //transaction.state.log.num.partitions
+            // try to speed up initial consumer group formation
+            .withEnv("KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS", "500") // group.initial.rebalance.delay.ms default: 3000
             .withReuse(true);
 
     static {
@@ -45,13 +49,13 @@ public abstract class KafkaTest<K, V> {
 
     protected KafkaClientUtils kcu = new KafkaClientUtils(kafkaContainer);
 
-  @BeforeAll
-  static void followKafkaLogs(){
-    if(log.isDebugEnabled()) {
-      FilteredTestContainerSlf4jLogConsumer logConsumer = new FilteredTestContainerSlf4jLogConsumer(log);
-      kafkaContainer.followOutput(logConsumer);
+    @BeforeAll
+    static void followKafkaLogs() {
+        if (log.isDebugEnabled()) {
+            FilteredTestContainerSlf4jLogConsumer logConsumer = new FilteredTestContainerSlf4jLogConsumer(log);
+            kafkaContainer.followOutput(logConsumer);
+        }
     }
-  }
 
     @BeforeEach
     void open() {
@@ -64,18 +68,30 @@ void close() {
     }
 
     void setupTopic() {
+        String name = LoadTest.class.getSimpleName();
+        setupTopic(name);
+    }
+
+    protected String setupTopic(String name) {
         assertThat(kafkaContainer.isRunning()).isTrue(); // sanity
 
-        topic = LoadTest.class.getSimpleName() + "-" + nextInt();
+        topic = name + "-" + nextInt();
 
         ensureTopic(topic, numPartitions);
+
+        return name;
     }
 
-    @SneakyThrows
     protected void ensureTopic(String topic, int numPartitions) {
         NewTopic e1 = new NewTopic(topic, numPartitions, (short) 1);
         CreateTopicsResult topics = kcu.admin.createTopics(UniLists.of(e1));
-        Void all = topics.all().get();
+        try {
+            Void all = topics.all().get();
+        } catch (ExecutionException e) {
+            // fine
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
     }
 
 }
diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/LoadTest.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/LoadTest.java
index 0e724c1f9..0bebc18bf 100644
--- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/LoadTest.java
+++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/LoadTest.java
@@ -30,6 +30,7 @@
 
 import static io.confluent.csid.utils.GeneralTestUtils.time;
 import static io.confluent.csid.utils.Range.range;
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.TRANSACTIONAL_PRODUCER;
 import static java.time.Duration.*;
 import static me.tongfei.progressbar.ProgressBar.wrap;
 import static org.apache.commons.lang3.RandomUtils.nextInt;
@@ -72,10 +73,11 @@ void asyncConsumeAndProcess() {
         kcu.consumer.subscribe(UniLists.of(topic));
 
         //
-        ParallelConsumerOptions options = ParallelConsumerOptions.builder().ordering(ParallelConsumerOptions.ProcessingOrder.KEY).build();
+        boolean tx = true;
+        ParallelConsumerOptions options = ParallelConsumerOptions.builder().ordering(ParallelConsumerOptions.ProcessingOrder.KEY).usingTransactionalProducer(tx).commitMode(TRANSACTIONAL_PRODUCER).build();
         KafkaConsumer<String, String> newConsumer = kcu.createNewConsumer();
         newConsumer.subscribe(Pattern.compile(topic));
-        var async = new ParallelEoSStreamProcessor<>(newConsumer, kcu.createNewProducer(true), options);
+        var async = new ParallelEoSStreamProcessor<>(newConsumer, kcu.createNewProducer(tx), options);
         AtomicInteger msgCount = new AtomicInteger(0);
 
         ProgressBar pb = new ProgressBarBuilder()
diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/TransactionAndCommitModeTest.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/TransactionAndCommitModeTest.java
new file mode 100644
index 000000000..d61d1af1d
--- /dev/null
+++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/TransactionAndCommitModeTest.java
@@ -0,0 +1,143 @@
+package io.confluent.parallelconsumer.integrationTests;
+
+/*-
+ * Copyright (C) 2020 Confluent, Inc.
+ */
+
+import io.confluent.parallelconsumer.ParallelConsumerOptions;
+import io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode;
+import io.confluent.parallelconsumer.ParallelEoSStreamProcessor;
+import io.confluent.parallelconsumer.integrationTests.KafkaTest;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.awaitility.core.ConditionTimeoutException;
+import org.junit.jupiter.api.RepeatedTest;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import pl.tlinkowski.unij.api.UniLists;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.TRANSACTIONAL_PRODUCER;
+import static java.time.Duration.ofSeconds;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+import static org.awaitility.Awaitility.waitAtMost;
+
+/**
+ * Originally created to reproduce bug #25 https://github.com/confluentinc/parallel-consumer/issues/25 which was a known
+ * issue with multi threaded use of the {@link org.apache.kafka.clients.producer.KafkaProducer}.
+ * <p>
+ * After fixing multi threading issue, using Producer transactions was made optional, and this test grew to uncover
+ * several issues with the new implementation of committing offsets through the {@link
+ * org.apache.kafka.clients.consumer.KafkaConsumer}.
+ *
+ * @see io.confluent.parallelconsumer.OffsetCommitter
+ * @see io.confluent.parallelconsumer.ConsumerOffsetCommitter
+ * @see io.confluent.parallelconsumer.ProducerManager
+ */
+@Slf4j
+public class TransactionAndCommitModeTest extends KafkaTest<String, String> {
+
+    int LOW_MAX_POLL_RECORDS_CONFIG = 1;
+    int DEFAULT_MAX_POLL_RECORDS_CONFIG = 500;
+    int HIGH_MAX_POLL_RECORDS_CONFIG = 10_000;
+
+    public List<String> processedAndProducedKeys = Collections.synchronizedList(new ArrayList<>());
+    public AtomicInteger processedCount = new AtomicInteger(0);
+    public AtomicInteger producedCount = new AtomicInteger(0);
+
+    // default
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    public void testDefaultMaxPoll(CommitMode commitMode) {
+        runTest(DEFAULT_MAX_POLL_RECORDS_CONFIG, commitMode);
+    }
+
+    @RepeatedTest(5)
+    public void testTransactionalDefaultMaxPoll() {
+        runTest(DEFAULT_MAX_POLL_RECORDS_CONFIG, TRANSACTIONAL_PRODUCER);
+    }
+
+    // low
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    public void testLowMaxPoll(CommitMode commitMode) {
+        runTest(LOW_MAX_POLL_RECORDS_CONFIG, commitMode);
+    }
+
+    // high counts
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    public void testHighMaxPollEnum(CommitMode commitMode) {
+        runTest(HIGH_MAX_POLL_RECORDS_CONFIG, commitMode);
+    }
+
+    private void runTest(int maxPoll, CommitMode commitMode) {
+        boolean tx = commitMode.equals(TRANSACTIONAL_PRODUCER);
+        runTest(tx, maxPoll, commitMode);
+    }
+
+    @SneakyThrows
+    private void runTest(boolean tx, int maxPoll, CommitMode commitMode) {
+        String inputName = setupTopic(this.getClass().getSimpleName() + "-input");
+        String outputName = setupTopic(this.getClass().getSimpleName() + "-input");
+
+        // pre-produce messages to input-topic
+        List<String> expectedKeys = new ArrayList<>();
+        int expectedMessageCount = 1000;
+        log.info("Producing {} messages before starting application", expectedMessageCount);
+        try (Producer<String, String> kafkaProducer = kcu.createNewProducer(false)) {
+            for (int i = 0; i < expectedMessageCount; i++) {
+                String key = "key-" + i;
+                kafkaProducer.send(new ProducerRecord<>(inputName, key, "value-" + i));
+                expectedKeys.add(key);
+            }
+        }
+
+        // run parallel-consumer
+        log.info("Starting application...");
+        ParallelEoSStreamProcessor<String, String> pc = new ParallelEoSStreamProcessor<>(kcu.consumer, kcu.producer, ParallelConsumerOptions.builder().build());
+        pc.subscribe(UniLists.of(inputName)); // <5>
+
+        pc.pollAndProduce(record -> {
+                    processedCount.incrementAndGet();
+                    return UniLists.of(new ProducerRecord<>(outputName, record.key(), "data"));
+                }, consumeProduceResult -> {
+                    producedCount.incrementAndGet();
+                    processedAndProducedKeys.add(consumeProduceResult.getIn().key());
+                }
+        );
+
+        // wait for all pre-produced messages to be processed and produced
+        String failureMessage = "All keys sent to input-topic should be processed and produced";
+        try {
+            waitAtMost(ofSeconds(10)).alias(failureMessage).untilAsserted(() -> {
+                log.debug("Processed-count: " + processedCount.get());
+                log.debug("Produced-count: " + producedCount.get());
+                List<String> processedAndProducedKeysCopy = new ArrayList<>(processedAndProducedKeys); // avoid concurrent-modification in assert
+                assertThat(processedAndProducedKeysCopy).contains(expectedKeys.toArray(new String[0]));
+            });
+        } catch (ConditionTimeoutException e) {
+            log.debug("Expected keys=" + expectedKeys + "");
+            log.debug("Processed and produced keys=" + processedAndProducedKeys + "");
+            expectedKeys.removeAll(processedAndProducedKeys);
+            log.debug("Missing keys=" + expectedKeys);
+            fail(failureMessage);
+        }
+
+        pc.closeDrainFirst();
+
+        assertThat(processedCount.get())
+                .as("messages processed and produced by parallel-consumer should be equal")
+                .isEqualTo(producedCount.get());
+
+        pc.close();
+    }
+
+}
diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VolumeTests.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VolumeTests.java
index 7c6a07612..df0f3c5c1 100644
--- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VolumeTests.java
+++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VolumeTests.java
@@ -4,6 +4,8 @@
  */
 package io.confluent.parallelconsumer.integrationTests;
 
+import io.confluent.parallelconsumer.ParallelConsumerOptions;
+import io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode;
 import io.confluent.parallelconsumer.ParallelEoSStreamProcessorTestBase;
 import io.confluent.csid.utils.KafkaTestUtils;
 import lombok.SneakyThrows;
@@ -16,8 +18,8 @@
 import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.common.TopicPartition;
 import org.assertj.core.util.Lists;
-import org.awaitility.Awaitility;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
 import org.testcontainers.shaded.com.google.common.collect.Maps;
 import pl.tlinkowski.unij.api.UniLists;
 
@@ -25,13 +27,17 @@
 import java.util.*;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.CountDownLatch;
+import java.util.stream.Collectors;
 
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.CONSUMER_SYNC;
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.TRANSACTIONAL_PRODUCER;
 import static io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder.*;
 import static io.confluent.csid.utils.GeneralTestUtils.time;
 import static io.confluent.csid.utils.Range.range;
 import static java.util.Comparator.comparing;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.waitAtMost;
 import static org.mockito.Mockito.mock;
 
 /**
@@ -43,10 +49,16 @@ public class VolumeTests extends ParallelEoSStreamProcessorTestBase {
     KafkaTestUtils ku = new KafkaTestUtils(consumerSpy);
 
     @SneakyThrows
-    @Test
-    public void load() {
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    public void load(CommitMode commitMode) {
         setupClients();
-        setupAsyncConsumerInstance(UNORDERED);
+        boolean useTxProducer = commitMode.equals(TRANSACTIONAL_PRODUCER);
+        setupParallelConsumerInstance(ParallelConsumerOptions.builder()
+                .ordering(UNORDERED)
+                .usingTransactionalProducer(useTxProducer)
+                .commitMode(commitMode)
+                .build());
 
 //        int quantityOfMessagesToProduce = 1_000_000;
         int quantityOfMessagesToProduce = 500;
@@ -54,42 +66,44 @@ public void load() {
         List<ConsumerRecord<String, String>> records = ku.generateRecords(quantityOfMessagesToProduce);
         ku.send(consumerSpy, records);
 
-        CountDownLatch latch = new CountDownLatch(quantityOfMessagesToProduce);
+        CountDownLatch allMessagesConsumedLatch = new CountDownLatch(quantityOfMessagesToProduce);
 
         parallelConsumer.pollAndProduce((rec) -> {
             ProducerRecord<String, String> mock = mock(ProducerRecord.class);
             return UniLists.of(mock);
         }, (x) -> {
 //            log.debug(x.toString());
-            latch.countDown();
+            allMessagesConsumedLatch.countDown();
         });
 
 
         //
+        allMessagesConsumedLatch.await(defaultTimeoutSeconds, SECONDS);
 //        waitAtMost(defaultTimeout).until(() -> producerSpy.consumerGroupOffsetsHistory().size() > 0);
-        latch.await(defaultTimeoutSeconds, SECONDS);
-        parallelConsumer.waitForNoInFlight(defaultTimeout.multipliedBy(10));
+        parallelConsumer.waitForProcessedNotCommitted(defaultTimeout.multipliedBy(10));
         parallelConsumer.close();
 
-
         // assert quantity of produced messages
         List<ProducerRecord<String, String>> history = producerSpy.history();
         assertThat(history).hasSize(quantityOfMessagesToProduce);
 
-        // assert order of produced messages
-
-
-        // assert order of commits
-//        int keySize = ku.getKeys().size();
-        List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> commitHistory = producerSpy.consumerGroupOffsetsHistory();
-        Map<String, Map<TopicPartition, OffsetAndMetadata>> mostRecent = commitHistory.get(commitHistory.size() - 1);
-        Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap = mostRecent.get(CONSUMER_GROUP_ID);
-        OffsetAndMetadata mostRecentTPCommit = topicPartitionOffsetAndMetadataMap.get(new TopicPartition(INPUT_TOPIC, 0));
-
-        assertCommitsAlwaysIncrease();
-
-        long mostRecentCommitOffset = findMostRecentCommitOffset(producerSpy);
-        assertThat(mostRecentCommitOffset).isEqualTo(quantityOfMessagesToProduce);
+        if (useTxProducer) {
+            // assert order of commits
+            assertCommitsAlwaysIncrease();
+
+            // producer (tx) commits
+            List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> producerCommits = producerSpy.consumerGroupOffsetsHistory();
+            assertThat(producerCommits).isNotEmpty();
+            long mostRecentProducerCommitOffset = findMostRecentCommitOffset(producerSpy); // tx
+            assertThat(mostRecentProducerCommitOffset).isEqualTo(quantityOfMessagesToProduce);
+        } else {
+            // assert commit messages
+            List<Map<TopicPartition, OffsetAndMetadata>> consumerCommitHistory = consumerSpy.getCommitHistoryInt();
+
+            assertThat(consumerCommitHistory).isNotEmpty();
+            long mostRecentConsumerCommitOffset = consumerCommitHistory.get(consumerCommitHistory.size() - 1).values().stream().collect(Collectors.toList()).get(0).offset(); // non-tx
+            assertThat(mostRecentConsumerCommitOffset).isEqualTo(quantityOfMessagesToProduce);
+        }
 
         // TODO: Assert process ordering
     }
@@ -114,6 +128,7 @@ private void assertCommitsAlwaysIncrease() {
 
     private long findMostRecentCommitOffset(MockProducer<?, ?> producerSpy) {
         List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> commitHistory = producerSpy.consumerGroupOffsetsHistory();
+        assertThat(commitHistory).as("No offsets committed").hasSizeGreaterThan(0);
         Map<String, Map<TopicPartition, OffsetAndMetadata>> mostRecent = commitHistory.get(commitHistory.size() - 1);
         Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap = mostRecent.get(CONSUMER_GROUP_ID);
         OffsetAndMetadata mostRecentTPCommit = topicPartitionOffsetAndMetadataMap.get(new TopicPartition(INPUT_TOPIC, 0));
@@ -126,41 +141,63 @@ private long findMostRecentCommitOffset(MockProducer<?, ?> producerSpy) {
      * <p>
      * Doesn't currently compare different key sizes, only partition order vs key order vs unordered
      */
-    @Test
-    public void timingOfDifferentOrderingTypes() {
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    public void timingOfDifferentOrderingTypes(CommitMode commitMode) {
         var quantityOfMessagesToProduce = 10_00;
         var defaultNumKeys = 20;
 
+        boolean useTxProducer = commitMode.equals(TRANSACTIONAL_PRODUCER);
+
+        ParallelConsumerOptions baseOptions = ParallelConsumerOptions.builder()
+                .ordering(UNORDERED)
+                .usingTransactionalProducer(useTxProducer)
+                .commitMode(commitMode)
+                .build();
+
+        setupParallelConsumerInstance(baseOptions);
+
         Duration unorderedDuration = null;
         for (var round : range(2)) { // warm up round first
-            setupAsyncConsumerInstance(UNORDERED);
+            setupParallelConsumerInstance(baseOptions.toBuilder().ordering(UNORDERED).build());
             log.debug("No order");
             unorderedDuration = time(() -> testTiming(defaultNumKeys, quantityOfMessagesToProduce));
             log.info("Duration for Unordered processing in round {} with {} keys was {}", round, defaultNumKeys, unorderedDuration);
         }
 
         var keyResults = Maps.<Integer, Duration>newTreeMap();
-        setupAsyncConsumerInstance(KEY);
         for (var keySize : UniLists.of(1, 2, 5, 10, 20, 50, 100, 1_000)) {
-            setupAsyncConsumerInstance(KEY);
+            setupParallelConsumerInstance(baseOptions.toBuilder().ordering(KEY).build());
             log.debug("By key, {} keys", keySize);
             Duration keyOrderDuration = time(() -> testTiming(keySize, quantityOfMessagesToProduce));
             log.info("Duration for Key order processing {} keys was {}", keySize, keyOrderDuration);
             keyResults.put(keySize, keyOrderDuration);
         }
 
-        setupAsyncConsumerInstance(PARTITION);
+        setupParallelConsumerInstance(baseOptions.toBuilder().ordering(PARTITION).build());
         log.debug("By partition");
         Duration partitionOrderDuration = time(() -> testTiming(defaultNumKeys, quantityOfMessagesToProduce));
         log.info("Duration for Partition order processing {} keys was {}", defaultNumKeys, partitionOrderDuration);
 
         log.info("Key duration results:\n{}", keyResults);
 
+        log.info("Unordered duration: {}", unorderedDuration);
+
         int numOfKeysToCompare = 5; // needs to be small enough that there's a significant difference between unordered and key of x
         Duration keyOrderHalfDefaultKeySize = keyResults.get(numOfKeysToCompare);
-        assertThat(unorderedDuration).as("UNORDERED is faster than PARTITION order").isLessThan(partitionOrderDuration);
-        assertThat(unorderedDuration).as("UNORDERED is faster than KEY order, keySize of: " + numOfKeysToCompare).isLessThan(keyOrderHalfDefaultKeySize);
-        assertThat(keyOrderHalfDefaultKeySize).as("KEY order is faster than PARTITION order").isLessThan(partitionOrderDuration);
+        assertThat(unorderedDuration).as("UNORDERED should be faster than PARTITION order")
+                .isLessThan(partitionOrderDuration);
+
+        if (commitMode.equals(CONSUMER_SYNC)) {
+            assertThat(unorderedDuration).as("Committing synchronously from the controller causes a large overhead, making UNORDERED very close in speed to KEY order, keySize of: " + numOfKeysToCompare)
+                    .isCloseTo(keyOrderHalfDefaultKeySize, keyOrderHalfDefaultKeySize.plus(keyOrderHalfDefaultKeySize.dividedBy(5))); // within 20%
+        } else {
+            assertThat(unorderedDuration).as("UNORDERED should be faster than KEY order, keySize of: " + numOfKeysToCompare)
+                    .isLessThan(keyOrderHalfDefaultKeySize);
+        }
+
+        assertThat(keyOrderHalfDefaultKeySize).as("KEY order is faster than PARTITION order")
+                .isLessThan(partitionOrderDuration);
     }
 
     /**
@@ -188,7 +225,6 @@ private void testTiming(int numberOfKeys, int quantityOfMessagesToProduce) {
             long sleepTime = (long) (Math.random() * rangeOfTimeSimulatedProcessingTakesMs);
             sleep(sleepTime);
             ProducerRecord<String, String> stub = new ProducerRecord<>(OUTPUT_TOPIC, "sk:" + rec.key(), "Processing took: " + sleepTime + ". SourceV:" + rec.value());
-//            ProducerRecord<String, String> stub = new ProducerRecord<>(OUTPUT_TOPIC, "sk:" + rec.key(), rec.value());
             bar.stepTo(producerSpy.history().size());
             return UniLists.of(stub);
         }, (x) -> {
@@ -196,7 +232,7 @@ private void testTiming(int numberOfKeys, int quantityOfMessagesToProduce) {
 //            log.debug(x.toString());
         });
 
-        Awaitility.waitAtMost(defaultTimeout.multipliedBy(10)).untilAsserted(() -> {
+        waitAtMost(defaultTimeout.multipliedBy(10)).untilAsserted(() -> {
             assertThat(super.successfulWork.size()).as("All messages expected messages were processed and successful").isEqualTo(quantityOfMessagesToProduce);
             assertThat(producerSpy.history().size()).as("All messages expected messages were processed and results produced").isEqualTo(quantityOfMessagesToProduce);
         });
@@ -205,8 +241,6 @@ private void testTiming(int numberOfKeys, int quantityOfMessagesToProduce) {
         log.info("Closing async client");
         parallelConsumer.close();
 
-        List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> groupOffsetsHistory = producerSpy.consumerGroupOffsetsHistory();
-
         assertCommitsAlwaysIncrease();
 
         //
@@ -250,7 +284,13 @@ private void testTiming(int numberOfKeys, int quantityOfMessagesToProduce) {
         }
 
         assertThat(producerSpy.history().size()).as("Finally, all messages expected messages were produced").isEqualTo(quantityOfMessagesToProduce);
-        assertThat(groupOffsetsHistory).as("No offsets committed").hasSizeGreaterThan(0);
+        if (isUsingTransactionalProducer()) {
+            List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> groupOffsetsHistory = producerSpy.consumerGroupOffsetsHistory(); // tx
+            assertThat(groupOffsetsHistory).as("No offsets committed").hasSizeGreaterThan(0); // tx
+        } else {
+            List<Map<TopicPartition, OffsetAndMetadata>> commitHistory = consumerSpy.getCommitHistoryInt();
+            assertThat(commitHistory).as("No offsets committed").hasSizeGreaterThan(0); // non-tx
+        }
 
         // clear messages
         super.successfulWork.clear();
diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/utils/KafkaClientUtils.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/utils/KafkaClientUtils.java
index ff9808c39..17ba965c3 100644
--- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/utils/KafkaClientUtils.java
+++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/utils/KafkaClientUtils.java
@@ -105,7 +105,12 @@ public <K, V> KafkaProducer<K, V> createNewProducer(boolean tx) {
         var txProps = new Properties();
         txProps.putAll(props);
         if (tx) {
-            txProps.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, this.getClass().getSimpleName()); // required for tx
+            // random number so we get a unique producer tx session each time. Normally wouldn't do this in production,
+            // but sometimes running in the test suite our producers step on each other between test runs and this causes
+            // Producer Fenced exceptions:
+            // Error looks like: Producer attempted an operation with an old epoch. Either there is a newer producer with
+            // the same transactionalId, or the producer's transaction has been expired by the broker.
+            txProps.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, this.getClass().getSimpleName() + ":" + RandomUtils.nextInt()); // required for tx
         }
         KafkaProducer<K, V> kvKafkaProducer = new KafkaProducer<>(txProps);
         log.debug("New producer {}", kvKafkaProducer);
diff --git a/parallel-consumer-core/src/test/java/io/confluent/csid/utils/KafkaTestUtils.java b/parallel-consumer-core/src/test/java/io/confluent/csid/utils/KafkaTestUtils.java
index b95a2ea6f..3dd09da7b 100644
--- a/parallel-consumer-core/src/test/java/io/confluent/csid/utils/KafkaTestUtils.java
+++ b/parallel-consumer-core/src/test/java/io/confluent/csid/utils/KafkaTestUtils.java
@@ -9,12 +9,12 @@
 import lombok.RequiredArgsConstructor;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.MockConsumer;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.producer.MockProducer;
 import org.apache.kafka.common.TopicPartition;
+import org.assertj.core.api.Assertions;
 import org.assertj.core.util.Lists;
 import pl.tlinkowski.unij.api.UniLists;
 
@@ -22,12 +22,11 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
+import static io.confluent.csid.utils.Range.range;
 import static io.confluent.parallelconsumer.ParallelEoSStreamProcessorTestBase.CONSUMER_GROUP_ID;
 import static io.confluent.parallelconsumer.ParallelEoSStreamProcessorTestBase.INPUT_TOPIC;
-import static io.confluent.csid.utils.Range.range;
 import static java.lang.Math.random;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.when;
 
 @Slf4j
 @RequiredArgsConstructor
@@ -37,11 +36,7 @@ public class KafkaTestUtils {
 
     int offset = 0;
 
-    public static final ConsumerGroupMetadata DEFAULT_GROUP_METADATA = new ConsumerGroupMetadata(CONSUMER_GROUP_ID);
-
-    public static void setupConsumer(MockConsumer mc) {
-        when(mc.groupMetadata()).thenReturn(DEFAULT_GROUP_METADATA);
-
+    public static void assignConsumerToTopic(final MockConsumer mc) {
         TopicPartition tp1 = new TopicPartition(INPUT_TOPIC, 1);
         TopicPartition tp0 = new TopicPartition(INPUT_TOPIC, 0);
         mc.assign(Lists.list(tp0, tp1));
@@ -91,29 +86,35 @@ public static void assertCommits(MockProducer mp, List<Integer> expectedOffsets,
                 .containsExactlyElementsOf(expectedOffsets);
     }
 
+    public static void assertCommitLists(MockProducer mp, List<List<Integer>> expectedPartitionOffsets, Optional<String> description) {
+        assertCommitLists(mp.consumerGroupOffsetsHistory(), expectedPartitionOffsets, description);
+    }
+
     /**
-     * Collects into a set - ignore repeated commits ({@link OffsetMapCodecManager})
+     * Collects into a set - ignore repeated commits ({@link OffsetMapCodecManager}).
+     *
+     * Ignores duplicates.
      *
      * @see OffsetMapCodecManager
      */
-    public static void assertCommitLists(MockProducer mp, List<List<Integer>> expectedPartitionOffsets, Optional<String> description) {
+    public static void assertCommitLists(List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> history, List<List<Integer>> expectedPartitionOffsets, Optional<String> description) {
         log.info("Asserting commits of {}", expectedPartitionOffsets);
-        List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> history = mp.consumerGroupOffsetsHistory();
 
         AtomicReference<String> topicName = new AtomicReference<>("");
-        var results = new HashMap<TopicPartition, Set<Integer>>(); // set - ignore repeated commits ({@link OffsetMap})
+        var partitionToCommittedOffsets = new HashMap<TopicPartition, Set<Integer>>(); // set - ignore repeated commits ({@link OffsetMap})
         history.stream().forEachOrdered(histories -> {
             // get all partition offsets and flatten
-            var group = histories.get(CONSUMER_GROUP_ID);
-            for (var actualPartitionOffsets : group.entrySet()) {
-                TopicPartition key = actualPartitionOffsets.getKey();
+            var partitionCommits = histories.get(CONSUMER_GROUP_ID);
+            for (var singlePartitionCommit : partitionCommits.entrySet()) {
+                TopicPartition key = singlePartitionCommit.getKey();
                 topicName.set(key.topic());
-                OffsetAndMetadata commit = actualPartitionOffsets.getValue();
+                OffsetAndMetadata commit = singlePartitionCommit.getValue();
                 int offset = (int) commit.offset();
-                results.computeIfAbsent(key, x -> new HashSet<>()).add(offset);
+                partitionToCommittedOffsets.computeIfAbsent(key, x -> new HashSet<>()).add(offset);
             }
         });
 
+        // compute the matching expected map
         var expectedMap = new HashMap<TopicPartition, Set<Integer>>();
         for (int i = 0; i < expectedPartitionOffsets.size(); i++) {
             List<Integer> offsets = expectedPartitionOffsets.get(i);
@@ -121,10 +122,17 @@ public static void assertCommitLists(MockProducer mp, List<List<Integer>> expect
             expectedMap.put(tp, new HashSet<>(offsets));
         }
 
-        assertThat(results).describedAs(description.orElse("Which offsets are committed and in the expected order"))
+        assertThat(partitionToCommittedOffsets).describedAs(description.orElse("Which offsets are committed and in the expected order"))
                 .containsExactlyEntriesOf(expectedMap);
     }
 
+    public static void assertLastCommitIs(final LongPollingMockConsumer<String, String> mockConsumer, final int expected) {
+        List<Map<TopicPartition, OffsetAndMetadata>> commits = mockConsumer.getCommitHistoryInt();
+        Assertions.assertThat(commits).isNotEmpty();
+        long offset = (int) commits.get(commits.size() - 1).values().iterator().next().offset();
+        Assertions.assertThat(offset).isEqualTo(expected);
+    }
+
     public List<ConsumerRecord<String, String>> generateRecords(int quantity) {
         HashMap<Integer, List<ConsumerRecord<String, String>>> integerListHashMap = generateRecords(defaultKeys, quantity);
         return flatten(integerListHashMap.values());
diff --git a/parallel-consumer-core/src/test/java/io/confluent/csid/utils/LongPollingMockConsumer.java b/parallel-consumer-core/src/test/java/io/confluent/csid/utils/LongPollingMockConsumer.java
index daebfe0b9..18629f957 100644
--- a/parallel-consumer-core/src/test/java/io/confluent/csid/utils/LongPollingMockConsumer.java
+++ b/parallel-consumer-core/src/test/java/io/confluent/csid/utils/LongPollingMockConsumer.java
@@ -4,12 +4,19 @@
  * Copyright (C) 2020 Confluent, Inc.
  */
 
+import lombok.Getter;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.MockConsumer;
-import org.apache.kafka.clients.consumer.OffsetResetStrategy;
+import org.apache.kafka.clients.consumer.*;
+import org.apache.kafka.clients.consumer.internals.SubscriptionState;
+import org.apache.kafka.clients.producer.MockProducer;
+import org.apache.kafka.common.TopicPartition;
 
+import java.lang.reflect.Field;
 import java.time.Duration;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 /**
  * Used in tests to stub out the behaviour of the real Broker and Client's long polling system (the mock Kafka Consumer
@@ -21,6 +28,9 @@
 @Slf4j
 public class LongPollingMockConsumer<K, V> extends MockConsumer<K, V> {
 
+    @Getter
+    private final List<Map<TopicPartition, OffsetAndMetadata>> commitHistoryInt = new ArrayList<>();
+
     public LongPollingMockConsumer(OffsetResetStrategy offsetResetStrategy) {
         super(offsetResetStrategy);
     }
@@ -53,5 +63,79 @@ public synchronized void wakeup() {
         }
     }
 
+    @Override
+    public synchronized void commitAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, final OffsetCommitCallback callback) {
+        commitHistoryInt.add(offsets);
+        super.commitAsync(offsets, callback);
+    }
+
+    /**
+     * Makes the commit history look like the {@link MockProducer}s one so we can use the same assert method.
+     *
+     * @see KafkaTestUtils#assertCommitLists(List, List, Optional)
+     */
+    private List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> injectConsumerGroupId(final List<Map<TopicPartition, OffsetAndMetadata>> commitHistory) {
+        String groupId = this.groupMetadata().groupId();
+        return commitHistory.stream()
+                .map(x -> {
+                    HashMap<String, Map<TopicPartition, OffsetAndMetadata>> stringMapHashMap = new HashMap<>();
+                    stringMapHashMap.put(groupId, x);
+                    return stringMapHashMap;
+                })
+                .collect(Collectors.toList());
+    }
+
+    /**
+     * Makes the commit history look like the {@link MockProducer}s one so we can use the same assert method.
+     *
+     * @see KafkaTestUtils#assertCommitLists(List, List, Optional)
+     */
+    public List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> getCommitHistoryWithGropuId() {
+        return injectConsumerGroupId(getCommitHistoryInt());
+    }
+
+    @Override
+    @SneakyThrows
+    public synchronized void close(final long timeout, final TimeUnit unit) {
+        revokeAssignment();
+        super.close(timeout, unit);
+    }
+
+    /**
+     * {@link MockConsumer} fails to implement any {@link ConsumerRebalanceListener} system, so we manually revoke
+     * assignments, use reflection to access the registered rebalance listener, call the listener, and only then close
+     * the consumer.
+     *
+     * @see io.confluent.parallelconsumer.ParallelEoSStreamProcessor#onPartitionsRevoked
+     */
+    private void revokeAssignment() throws NoSuchFieldException, IllegalAccessException {
+        // access listener
+        Field subscriptionsField = MockConsumer.class.getDeclaredField("subscriptions"); //NoSuchFieldException
+        subscriptionsField.setAccessible(true);
+        SubscriptionState subscriptionState = (SubscriptionState) subscriptionsField.get(this); //IllegalAccessException
+        ConsumerRebalanceListener consumerRebalanceListener = subscriptionState.rebalanceListener();
+
+        // execute
+        if (consumerRebalanceListener == null) {
+            log.debug("No rebalance listener assigned - on revoke can't fire");
+        } else {
+            Set<TopicPartition> assignment = super.assignment();
+            consumerRebalanceListener.onPartitionsRevoked(assignment);
+        }
+    }
+
+    public void subscribeWithRebalanceAndAssignment(final List<String> topics, int partitions) {
+        List<TopicPartition> topicPartitions = topics.stream()
+                .flatMap(y -> Range.rangeStream(partitions).boxed()
+                        .map(x -> new TopicPartition(y, x)))
+                .collect(Collectors.toList());
+        super.rebalance(topicPartitions);
+        //
+        HashMap<TopicPartition, Long> beginningOffsets = new HashMap<>();
+        for (var tp : topicPartitions) {
+            beginningOffsets.put(tp, 0L);
+        }
+        super.updateBeginningOffsets(beginningOffsets);
 
-}
+    }
+}
\ No newline at end of file
diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTest.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTest.java
index 923ba0a4c..fbef02bdc 100644
--- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTest.java
+++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTest.java
@@ -4,20 +4,20 @@
  * Copyright (C) 2020 Confluent, Inc.
  */
 
-import io.confluent.csid.utils.WallClock;
+import io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.ConsumerRecords;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.clients.producer.RecordMetadata;
 import org.apache.kafka.common.TopicPartition;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
@@ -26,12 +26,13 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.function.Function;
 
-import static io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder.KEY;
 import static io.confluent.csid.utils.GeneralTestUtils.time;
 import static io.confluent.csid.utils.KafkaUtils.toTP;
 import static io.confluent.csid.utils.Range.range;
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.TRANSACTIONAL_PRODUCER;
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder.KEY;
 import static java.time.Duration.ofMillis;
-
+import static java.time.Duration.ofSeconds;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.assertj.core.api.Assertions.*;
 import static org.awaitility.Awaitility.await;
@@ -58,9 +59,12 @@ public void setupData() {
         primeFirstRecord();
     }
 
-    @Test
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
     @SneakyThrows
-    public void failingActionNothingCommitted() {
+    public void failingActionNothingCommitted(CommitMode commitMode) {
+        setupParallelConsumerInstance(commitMode);
+
         parallelConsumer.poll((ignore) -> {
             throw new RuntimeException("My user's function error");
         });
@@ -71,7 +75,8 @@ public void failingActionNothingCommitted() {
 
         //
         assertCommits(of(0), "All erroring, nothing committed except initial");
-        List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> maps = producerSpy.consumerGroupOffsetsHistory();
+        List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> maps = getCommitHistory();
+        assertThat(maps).isNotEmpty();
         List<OffsetAndMetadata> metas = new ArrayList<>();
         for (final Map<String, Map<TopicPartition, OffsetAndMetadata>> map : maps) {
             for (final Map<TopicPartition, OffsetAndMetadata> value : map.values()) {
@@ -88,9 +93,12 @@ public void failingActionNothingCommitted() {
         }
     }
 
-    @Test
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
     @SneakyThrows
-    public void offsetsAreNeverCommittedForMessagesStillInFlightSimplest() {
+    public void offsetsAreNeverCommittedForMessagesStillInFlightSimplest(CommitMode commitMode) {
+        setupParallelConsumerInstance(commitMode);
+
         sendSecondRecord(consumerSpy);
 
         var locks = constructLatches(2);
@@ -117,7 +125,7 @@ public void offsetsAreNeverCommittedForMessagesStillInFlightSimplest() {
         // make sure no offsets are committed
         assertCommits(of(0), "Partition is blocked");
 
-        // So it's data setup can be used in other tests, finish 0
+        // So it's data is setup can be used in other tests, finish 0
         releaseAndWait(locks, 0);
 
         log.debug("Closing...");
@@ -126,26 +134,36 @@ public void offsetsAreNeverCommittedForMessagesStillInFlightSimplest() {
         assertThat(processedStates.values()).as("sanity - all expected messages are processed").containsExactly(true, true);
     }
 
-    @Test
-    public void offsetsAreNeverCommittedForMessagesStillInFlightShort() {
-        offsetsAreNeverCommittedForMessagesStillInFlightSimplest();
+    private void setupParallelConsumerInstance(final CommitMode commitMode) {
+        setupParallelConsumerInstance(getBaseOptions(commitMode));
+        // created a new client above, so have to send the prime record again
+        primeFirstRecord();
+    }
 
-        log.info("Test start");
+    private ParallelConsumerOptions getBaseOptions(final CommitMode commitMode) {
+        return ParallelConsumerOptions.builder()
+                .commitMode(commitMode)
+                .usingTransactionalProducer(commitMode.equals(TRANSACTIONAL_PRODUCER))
+                .build();
+    }
 
-        // make sure offset 1, not 0 is committed
-        // check only 1 is now committed, not committing 0 as well is a performance thing
-        verify(producerSpy,
-                after(verificationWaitDelay)
-                        .atLeast(1)
-                        .description("wait for at least one commit call"))
-                .commitTransaction();
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    @SneakyThrows
+    public void offsetsAreNeverCommittedForMessagesStillInFlightShort(CommitMode commitMode) {
+        offsetsAreNeverCommittedForMessagesStillInFlightSimplest(commitMode);
+        log.info("Test start");
 
+        // next expected offset is now 2
         assertCommits(of(0, 2), "Only one of the two offsets committed, as they were coalesced for efficiency");
     }
 
     @Disabled
-    @Test
-    public void offsetsAreNeverCommittedForMessagesStillInFlightLong() {
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    public void offsetsAreNeverCommittedForMessagesStillInFlightLong(CommitMode commitMode) {
+        setupParallelConsumerInstance(commitMode);
+
         sendSecondRecord(consumerSpy);
 
         // send three messages - 0, 1, 2
@@ -224,8 +242,14 @@ public void offsetsAreNeverCommittedForMessagesStillInFlightLong() {
         parallelConsumer.close();
     }
 
-    @Test
-    public void offsetCommitsAreIsolatedPerPartition() {
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    @SneakyThrows
+    public void offsetCommitsAreIsolatedPerPartition(CommitMode commitMode) {
+        setupParallelConsumerInstance(commitMode);
+
+//        @Test
+//    public void offsetCommitsAreIsolatedPerPartition() {
         sendSecondRecord(consumerSpy);
 
         // send three messages - 0,1, to one partition and 3,4 to another partition petitions
@@ -252,6 +276,8 @@ public void offsetCommitsAreIsolatedPerPartition() {
         // finish processing 1
         releaseAndWait(locks, 1);
 
+        waitForSomeLoopCycles(5);
+
         // make sure only base offsets are committed
         assertCommits(of(0, 2));
         assertCommitLists(of(of(0), of(2)));
@@ -267,12 +293,20 @@ public void offsetCommitsAreIsolatedPerPartition() {
         // finish 0
         releaseAndWait(locks, 0);
 
+        // async consumer is slower to execute the commit. We could just wait, or we could add an event to the async consumer commit cycle
+        if (isUsingAsyncCommits())
+            waitForOneLoopCycle();
+
         // make sure offset 0 and 1 is committed
         assertCommitLists(of(of(0, 2), of(2, 3)));
 
         // finish 3
         releaseAndWait(locks, 3);
 
+        // async consumer is slower to execute the commit. We could just wait, or we could add an event to the async consumer commit cycle
+        if (isUsingAsyncCommits())
+            waitForOneLoopCycle();
+
         //
         assertCommitLists(of(of(0, 2), of(2, 3, 4)));
     }
@@ -288,28 +322,39 @@ public void avro() {
         assertThat(false).isTrue();
     }
 
-    @Test
-    @SneakyThrows
-    public void controlFlowException() {
-        //
-        WallClock mock = mock(WallClock.class);
-        when(mock.getNow()).thenThrow(new RuntimeException("My fake control loop error"));
-        parallelConsumer.setClock(mock);
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    public void controlFlowException(CommitMode commitMode) {
+        // setup again manually to use subscribe instead of assign (for revoke testing)
+        instantiateConsumerProducer();
+        parallelConsumer = initAsyncConsumer(getBaseOptions(commitMode));
+        subscribeParallelConsumerAndMockConsumerTo(INPUT_TOPIC);
+        setupData();
+
+
+        // cause a control loop error
+        parallelConsumer.addLoopEndCallBack(() -> {
+            throw new RuntimeException("My fake control loop error");
+        });
+
 
         //
         parallelConsumer.poll((ignore) -> {
-            // ignore
+            log.info("Ignoring {}", ignore);
         });
 
         // close and retrieve exception in control loop
         assertThatThrownBy(() -> {
-            parallelConsumer.closeDontDrainFirst();
+            parallelConsumer.closeDrainFirst(ofSeconds(10));
         }).hasMessageContainingAll("Error", "poll", "thread", "fake control");
     }
 
-    @Test
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
     @SneakyThrows
-    public void testVoid() {
+    public void testVoid(CommitMode commitMode) {
+        setupParallelConsumerInstance(commitMode);
+
         int expected = 1;
         var msgCompleteBarrier = new CountDownLatch(expected);
         parallelConsumer.poll((record) -> {
@@ -322,11 +367,18 @@ public void testVoid() {
 
         waitForSomeLoopCycles(1);
 
+        parallelConsumer.close();
+
         assertCommits(of(0, 1));
 
         verify(myRecordProcessingAction, times(expected)).apply(any());
-        verify(producerSpy, atLeastOnce()).commitTransaction();
-        verify(producerSpy, atLeastOnce()).sendOffsetsToTransaction(anyMap(), ArgumentMatchers.<ConsumerGroupMetadata>any());
+
+        // assert internal methods - shouldn't really need this as we already check the commit history above through the
+        // spy, so can leave in for the old producer style
+        if (commitMode.equals(TRANSACTIONAL_PRODUCER)) {
+            verify(producerSpy, atLeastOnce()).commitTransaction();
+            verify(producerSpy, atLeastOnce()).sendOffsetsToTransaction(anyMap(), ArgumentMatchers.<ConsumerGroupMetadata>any());
+        }
     }
 
     /**
@@ -339,16 +391,6 @@ private void waitForInitialBootstrapCommit() {
                 .untilAsserted(() -> assertCommits(of(0)));
     }
 
-    @Test
-    public void testProducerStep() {
-        ProducerRecord<String, String> outMsg = new ProducerRecord(OUTPUT_TOPIC, "");
-        RecordMetadata prodResult = parallelConsumer.produceMessage(outMsg);
-        assertThat(prodResult).isNotNull();
-
-        List<ProducerRecord<String, String>> history = producerSpy.history();
-        assertThat(history).hasSize(1);
-    }
-
     @Test
     @Disabled
     public void userSucceedsButProduceToBrokerFails() {
@@ -381,18 +423,22 @@ public void failingMessagesThatAreRetriedDontBreakProcessingOrders() {
     public void ifTooManyMessagesAreInFlightDontPollBrokerForMore() {
     }
 
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
     @SneakyThrows
-    @Test
     @Disabled
-    public void processInKeyOrder() {
-        ParallelConsumerOptions options = ParallelConsumerOptions.builder().ordering(KEY).build();
-        setupAsyncConsumerInstance(options);
+    public void processInKeyOrder(CommitMode commitMode) {
+        setupParallelConsumerInstance(ParallelConsumerOptions.builder()
+                .commitMode(commitMode)
+                .ordering(KEY)
+                .usingTransactionalProducer(commitMode.equals(TRANSACTIONAL_PRODUCER))
+                .build());
+        // created a new client above, so have to send the prime record again
+        primeFirstRecord();
 
         // sanity check
         assertThat(parallelConsumer.wm.getOptions().getOrdering()).isEqualTo(KEY);
 
-        primeFirstRecord();
-
         sendSecondRecord(consumerSpy);
 
         // 0,1 previously sent to partition 0
@@ -534,7 +580,7 @@ public void processInKeyOrder() {
     @Test
     public void processInKeyOrderWorkNotReturnedDoesntBreakCommits() {
         ParallelConsumerOptions options = ParallelConsumerOptions.builder().ordering(KEY).build();
-        setupAsyncConsumerInstance(options);
+        setupParallelConsumerInstance(options);
         primeFirstRecord();
 
         sendSecondRecord(consumerSpy);
@@ -612,8 +658,11 @@ public void processInKeyOrderWorkNotReturnedDoesntBreakCommits() {
         }
     }
 
-    @Test
-    public void closeAfterSingleMessageShouldBeEventBasedFast() {
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    public void closeAfterSingleMessageShouldBeEventBasedFast(CommitMode commitMode) {
+        setupParallelConsumerInstance(commitMode);
+
         var msgCompleteBarrier = new CountDownLatch(1);
 
         parallelConsumer.poll((ignore) -> {
@@ -627,6 +676,10 @@ public void closeAfterSingleMessageShouldBeEventBasedFast() {
         // allow for offset to be committed
         waitForOneLoopCycle();
 
+        if (isUsingAsyncCommits()) {
+            waitForOneLoopCycle();
+        }
+
         assertCommits(of(0, 1));
 
         // close
@@ -639,8 +692,11 @@ public void closeAfterSingleMessageShouldBeEventBasedFast() {
                 .isLessThan(ofMillis(500));
     }
 
-    @Test
-    public void closeWithoutRunningShouldBeEventBasedFast() {
+    @ParameterizedTest()
+    @EnumSource(CommitMode.class)
+    public void closeWithoutRunningShouldBeEventBasedFast(CommitMode commitMode) {
+        setupParallelConsumerInstance(getBaseOptions(commitMode));
+
         parallelConsumer.closeDontDrainFirst();
     }
 
diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTestBase.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTestBase.java
index c20cbdbbe..f02b79fb2 100644
--- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTestBase.java
+++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTestBase.java
@@ -6,28 +6,33 @@
 
 import io.confluent.csid.utils.KafkaTestUtils;
 import io.confluent.csid.utils.LongPollingMockConsumer;
+import io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.kafka.clients.consumer.*;
 import org.apache.kafka.clients.producer.MockProducer;
 import org.apache.kafka.common.TopicPartition;
+import org.assertj.core.util.Lists;
 import org.junit.jupiter.api.BeforeEach;
+import pl.tlinkowski.unij.api.UniLists;
 import pl.tlinkowski.unij.api.UniMaps;
 
 import java.time.Duration;
 import java.util.*;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import static io.confluent.csid.utils.Range.range;
+import static io.confluent.csid.utils.StringUtils.msg;
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.*;
 import static java.time.Duration.ofMillis;
 import static java.time.Duration.ofSeconds;
 import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Awaitility.waitAtMost;
 import static org.mockito.Mockito.*;
+import static pl.tlinkowski.unij.api.UniLists.of;
 
 @Slf4j
 public class ParallelEoSStreamProcessorTestBase {
@@ -36,24 +41,27 @@ public class ParallelEoSStreamProcessorTestBase {
     public static final String OUTPUT_TOPIC = "output";
     public static final String CONSUMER_GROUP_ID = "my-group";
 
+    public static final ConsumerGroupMetadata DEFAULT_GROUP_METADATA = new ConsumerGroupMetadata(CONSUMER_GROUP_ID);
+
     /**
      * The frequency with which we pretend to poll the broker for records - actually the pretend long poll timeout. A
      * lower value shouldn't affect test speed much unless many different batches of messages are "published".
+     *
      * @see LongPollingMockConsumer#poll(Duration)
      */
-    public static final int DEFAULT_BROKER_POLL_FREQUENCY_MS = 100;
+    public static final int DEFAULT_BROKER_POLL_FREQUENCY_MS = 1000;
 
     /**
-     * The commit interval for the main {@link ParallelEoSStreamProcessor} control thread. Actually the timeout that we poll the
-     * {@link LinkedBlockingQueue} for. A lower value will increase the frequency of control loop cycles, making our
-     * test waiting go faster.
+     * The commit interval for the main {@link ParallelEoSStreamProcessor} control thread. Actually the timeout that we
+     * poll the {@link LinkedBlockingQueue} for. A lower value will increase the frequency of control loop cycles,
+     * making our test waiting go faster.
      *
      * @see ParallelEoSStreamProcessor#workMailBox
      * @see ParallelEoSStreamProcessor#processWorkCompleteMailBox
      */
-    public static final int DEFAULT_COMMIT_INTERVAL_MAX_MS = 100;
+    public static final int DEFAULT_COMMIT_INTERVAL_MAX_MS = 1000;
 
-    protected MockConsumer<String, String> consumerSpy;
+    protected LongPollingMockConsumer<String, String> consumerSpy;
     protected MockProducer<String, String> producerSpy;
 
     protected ParallelEoSStreamProcessor<String, String> parallelConsumer;
@@ -62,7 +70,7 @@ public class ParallelEoSStreamProcessorTestBase {
 
     protected static Duration defaultTimeout = ofSeconds(defaultTimeoutSeconds);
     protected static long defaultTimeoutMs = defaultTimeout.toMillis();
-    protected static Duration infiniteTimeout = Duration.ofMinutes(20);
+    protected static Duration effectivelyInfiniteTimeout = Duration.ofMinutes(20);
 
     ParallelEoSStreamProcessorTest.MyAction myRecordProcessingAction;
 
@@ -84,13 +92,13 @@ public class ParallelEoSStreamProcessorTestBase {
 
     @BeforeEach
     public void setupAsyncConsumerTestBase() {
-        setupAsyncConsumerInstance(ParallelConsumerOptions.builder().build());
+        setupParallelConsumerInstance(ParallelConsumerOptions.builder().commitMode(CONSUMER_SYNC).build());
     }
 
     protected List<WorkContainer<String, String>> successfulWork = Collections.synchronizedList(new ArrayList<>());
 
     private void setupWorkManager(WorkManager<String, String> wm) {
-        wm.getSuccessfulWorkListeners().add((work)->{
+        wm.getSuccessfulWorkListeners().add((work) -> {
             log.debug("Test work listener heard some successful work: {}", work);
             successfulWork.add(work);
         });
@@ -102,25 +110,41 @@ protected void primeFirstRecord() {
     }
 
     protected MockConsumer<String, String> setupClients() {
-        MockConsumer<String, String> consumer = new LongPollingMockConsumer<>(OffsetResetStrategy.EARLIEST);
+        instantiateConsumerProducer();
+
+        ktu = new KafkaTestUtils(consumerSpy);
+
+        KafkaTestUtils.assignConsumerToTopic(this.consumerSpy);
+
+        return consumerSpy;
+    }
+
+    protected void instantiateConsumerProducer() {
+        LongPollingMockConsumer<String, String> consumer = new LongPollingMockConsumer<>(OffsetResetStrategy.EARLIEST);
         MockProducer<String, String> producer = new MockProducer<>(true, null, null); // TODO do async testing
 
         this.producerSpy = spy(producer);
         this.consumerSpy = spy(consumer);
         myRecordProcessingAction = mock(ParallelEoSStreamProcessorTest.MyAction.class);
 
-        ktu = new KafkaTestUtils(consumerSpy);
-
-        KafkaTestUtils.setupConsumer(this.consumerSpy);
+        when(consumerSpy.groupMetadata()).thenReturn(DEFAULT_GROUP_METADATA);
+    }
 
-        return consumerSpy;
+    /**
+     * Need to make sure we only use {@link ParallelEoSStreamProcessor#subscribe} methods, and not do manual assignment,
+     * otherwise rebalance listeneres don't fire (because there are never rebalances).
+     */
+    protected void subscribeParallelConsumerAndMockConsumerTo(String topic) {
+        List<String> of = of(topic);
+        parallelConsumer.subscribe(of);
+        consumerSpy.subscribeWithRebalanceAndAssignment(of, 2);
     }
 
-    protected void setupAsyncConsumerInstance(ParallelConsumerOptions.ProcessingOrder order) {
-        setupAsyncConsumerInstance(ParallelConsumerOptions.builder().ordering(order).build());
+    protected void setupParallelConsumerInstance(ProcessingOrder order) {
+        setupParallelConsumerInstance(ParallelConsumerOptions.builder().ordering(order).build());
     }
 
-    protected void setupAsyncConsumerInstance(ParallelConsumerOptions parallelConsumerOptions) {
+    protected void setupParallelConsumerInstance(ParallelConsumerOptions parallelConsumerOptions) {
         setupClients();
 
         parallelConsumer = initAsyncConsumer(parallelConsumerOptions);
@@ -203,7 +227,9 @@ protected void waitUntilTrue(Callable<Boolean> booleanCallable) {
     private void blockingLoopLatchTrigger(int waitForCount) {
         log.debug("Waiting on {} cycles on loop latch...", waitForCount);
         loopLatchV = new CountDownLatch(waitForCount);
-        loopLatchV.await(defaultTimeoutSeconds, TimeUnit.SECONDS);
+        boolean await = loopLatchV.await(defaultTimeoutSeconds, SECONDS);
+        if (!await)
+            throw new TimeoutException(msg("Timeout {} waiting for latch", defaultTimeoutSeconds));
     }
 
     @SneakyThrows
@@ -222,16 +248,76 @@ protected void waitForCommitExact(int partition, int offset) {
                 any(ConsumerGroupMetadata.class));
     }
 
-    public void assertCommits(List<Integer> integers, String description) {
-        KafkaTestUtils.assertCommits(producerSpy, integers, Optional.of(description));
+    public void assertCommits(List<Integer> offsets, String description) {
+        assertCommits(offsets, Optional.of(description));
+    }
+
+    /**
+     * Flattens the offsets of all partitions into a single sequential list
+     */
+    public void assertCommits(List<Integer> offsets, Optional<String> description) {
+        if (isUsingTransactionalProducer()) {
+            KafkaTestUtils.assertCommits(producerSpy, offsets, description);
+            assertThat(extractAllPartitionsOffsetsSequentially()).isEmpty();
+        } else {
+            List<Integer> collect = extractAllPartitionsOffsetsSequentially();
+            // duplicates are ok
+            // is there a nicer optional way?
+            // {@link Optional#ifPresentOrElse} only @since 9
+            if (description.isPresent()) {
+                assertThat(collect).as(description.get()).hasSameElementsAs(offsets);
+            } else {
+                assertThat(collect).hasSameElementsAs(offsets);
+            }
+
+            KafkaTestUtils.assertCommits(producerSpy, UniLists.of(), Optional.of("Empty"));
+        }
+    }
+
+    /**
+     * Flattens the offsets of all partitions into a single sequential list
+     */
+    private List<Integer> extractAllPartitionsOffsetsSequentially() {
+        var result = new ArrayList<Integer>();
+        // copy the list for safe concurrent access
+        List<Map<TopicPartition, OffsetAndMetadata>> history = new ArrayList<>(consumerSpy.getCommitHistoryInt());
+        return history.stream()
+                .flatMap(commits ->
+                        commits.values().stream().map(meta -> (int) meta.offset())
+                ).collect(Collectors.toList());
+    }
+
+    public void assertCommits(List<Integer> offsets) {
+        assertCommits(offsets, Optional.empty());
+    }
+
+    /**
+     * Checks a list of commits of a list of partitions
+     */
+    public void assertCommitLists(List<List<Integer>> offsets) {
+        if (isUsingTransactionalProducer()) {
+            KafkaTestUtils.assertCommitLists(producerSpy, offsets, Optional.empty());
+        } else {
+            KafkaTestUtils.assertCommitLists(consumerSpy.getCommitHistoryWithGropuId(), offsets, Optional.empty());
+        }
+    }
+
+    protected List<Map<String, Map<TopicPartition, OffsetAndMetadata>>> getCommitHistory() {
+        if (isUsingTransactionalProducer()) {
+            return producerSpy.consumerGroupOffsetsHistory();
+        } else {
+            return consumerSpy.getCommitHistoryWithGropuId();
+        }
     }
 
-    public void assertCommits(List<Integer> integers) {
-        KafkaTestUtils.assertCommits(producerSpy, integers, Optional.empty());
+    protected boolean isUsingTransactionalProducer() {
+        ParallelConsumerOptions.CommitMode commitMode = parallelConsumer.getWm().getOptions().getCommitMode();
+        return commitMode.equals(TRANSACTIONAL_PRODUCER);
     }
 
-    public void assertCommitLists(List<List<Integer>> integers) {
-        KafkaTestUtils.assertCommitLists(producerSpy, integers, Optional.empty());
+    protected boolean isUsingAsyncCommits() {
+        ParallelConsumerOptions.CommitMode commitMode = parallelConsumer.getWm().getOptions().getCommitMode();
+        return commitMode.equals(CONSUMER_ASYNCHRONOUS);
     }
 
     protected void awaitLatch(List<CountDownLatch> latches, int latchIndex) {
diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/WorkManagerTest.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/WorkManagerTest.java
index 1322608a8..1dc21746c 100644
--- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/WorkManagerTest.java
+++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/WorkManagerTest.java
@@ -318,7 +318,7 @@ public void maxPerTopic() {
     public void maxInFlight() {
         //
         var opts = ParallelConsumerOptions.builder();
-        opts.maxUncommittedMessagesToHandlePerPartition(1);
+        opts.maxUncommittedMessagesToHandle(1);
         setupWorkManager(opts.build());
 
         //
@@ -372,7 +372,7 @@ public void maxConcurrencyVsInFlightAndNoLeaks() {
         var opts = ParallelConsumerOptions.builder();
         opts.ordering(UNORDERED);
 
-        opts.maxUncommittedMessagesToHandlePerPartition(3);
+        opts.maxUncommittedMessagesToHandle(3);
         opts.maxConcurrency(2);
 
         setupWorkManager(opts.build());
@@ -610,7 +610,7 @@ public void workQueuesEmptyWhenAllWorkComplete() {
         ParallelConsumerOptions build = ParallelConsumerOptions.builder()
                 .ordering(UNORDERED)
                 .maxConcurrency(10)
-                .maxUncommittedMessagesToHandlePerPartition(10)
+                .maxUncommittedMessagesToHandle(10)
                 .build();
         setupWorkManager(build);
         registerSomeWork();
diff --git a/parallel-consumer-core/src/test/resources/junit-platform.properties b/parallel-consumer-core/src/test/resources/junit-platform.properties
index 7f40a1ce1..e3c240beb 100644
--- a/parallel-consumer-core/src/test/resources/junit-platform.properties
+++ b/parallel-consumer-core/src/test/resources/junit-platform.properties
@@ -2,4 +2,4 @@
 # Copyright (C) 2020 Confluent, Inc.
 #
 
-junit.jupiter.displayname.generator.default = io.confluent.csid.utils.ReplaceCamelCase
\ No newline at end of file
+#junit.jupiter.displayname.generator.default = io.confluent.csid.utils.ReplaceCamelCase
\ No newline at end of file
diff --git a/parallel-consumer-core/src/test/resources/logback-test.xml b/parallel-consumer-core/src/test/resources/logback-test.xml
index aa004bb17..9f4330bfe 100644
--- a/parallel-consumer-core/src/test/resources/logback-test.xml
+++ b/parallel-consumer-core/src/test/resources/logback-test.xml
@@ -27,29 +27,30 @@
 
     <!-- primary -->
     <logger name="io.confluent.parallelconsumer" level="info"/>
-<!--    <logger name="io.confluent.parallelconsumer" level="debug"/>-->
+    <logger name="io.confluent.parallelconsumer" level="debug"/>
 <!--    <logger name="io.confluent.parallelconsumer" level="trace"/>-->
 <!--        <logger name="io.confluent.parallelconsumer.ParallelEoSStreamProcessor" level="trace"/>-->
 <!--    <logger name="io.confluent.parallelconsumer.integrationTests.KafkaTest" level="info"/> &lt;!&ndash; docker logs &ndash;&gt;-->
     <logger name="io.confluent.csid" level="info"/>
 <!--    <logger name="io.confluent.csid" level="debug"/>-->
-<!--            <logger name="io.confluent.csid" level="trace" />-->
+
+<!--            <logger name="io.confluent.parallelconsumer" level="trace" />-->
 
     <!-- notable specifics -->
-    <!--    <logger name="io.confluent.csid.asyncconsumer.integrationTests.VolumeTests" level="debug"/>-->
-    <!--    <logger name="io.confluent.csid.asyncconsumer.ParallelConsumerTestBase" level="trace"/>-->
-    <!--    <logger name="io.confluent.csid.asyncconsumer.WorkManager" level="trace" />-->
-    <!--    <logger name="io.confluent.csid.asyncconsumer.ParallelConsumer" level="info" />-->
-    <!--    <logger name="io.confluent.csid.asyncconsumer.OffsetMapCodecManager" level="debug"/>-->
-    <!--    <logger name="io.confluent.csid.asyncconsumer.WorkManagerOffsetMapCodecManagerTest" level="debug"/>-->
+    <!--    <logger name="io.confluent.parallelconsumer.integrationTests.VolumeTests" level="debug"/>-->
+    <!--    <logger name="io.confluent.parallelconsumer.ParallelConsumerTestBase" level="trace"/>-->
+    <!--    <logger name="io.confluent.parallelconsumer.WorkManager" level="trace" />-->
+    <!--    <logger name="io.confluent.parallelconsumer.ParallelConsumer" level="info" />-->
+    <!--    <logger name="io.confluent.parallelconsumer.OffsetMapCodecManager" level="debug"/>-->
+    <!--    <logger name="io.confluent.parallelconsumer.WorkManagerOffsetMapCodecManagerTest" level="debug"/>-->
 
     <!-- containers -->
-    <!--    <logger name="io.confluent.csid.asyncconsumer.integrationTests.DbTest" level="debug"/>-->
-    <!--    <logger name="io.confluent.csid.asyncconsumer.integrationTests.KafkaTest" level="debug"/>-->
+    <!--    <logger name="io.confluent.parallelconsumer.integrationTests.DbTest" level="debug"/>-->
+    <!--    <logger name="io.confluent.parallelconsumer.integrationTests.KafkaTest" level="debug"/>-->
 
     <!-- related -->
+<!--    <logger name="org.apache.kafka.clients" level="trace"/>-->
     <logger name="org.apache.kafka.clients" level="warn"/>
-<!--    <logger name="org.apache.kafka.clients" level="info"/>-->
 <!--    <logger name="org.apache.kafka.clients" level="debug"/>-->
 
     <logger name="org.postgresql" level="info"/>
diff --git a/parallel-consumer-examples/parallel-consumer-example-core/src/main/java/io/confluent/parallelconsumer/examples/core/CoreApp.java b/parallel-consumer-examples/parallel-consumer-example-core/src/main/java/io/confluent/parallelconsumer/examples/core/CoreApp.java
index ee01ee717..97a1586b2 100644
--- a/parallel-consumer-examples/parallel-consumer-example-core/src/main/java/io/confluent/parallelconsumer/examples/core/CoreApp.java
+++ b/parallel-consumer-examples/parallel-consumer-example-core/src/main/java/io/confluent/parallelconsumer/examples/core/CoreApp.java
@@ -18,7 +18,11 @@
 import org.apache.kafka.clients.producer.ProducerRecord;
 import pl.tlinkowski.unij.api.UniLists;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder.KEY;
 
@@ -28,8 +32,9 @@
 @Slf4j
 public class CoreApp {
 
-    static String inputTopic = "input-topic-" + RandomUtils.nextInt();
-    static String outputTopic = "output-topic-" + RandomUtils.nextInt();
+
+    String inputTopic = "input-topic-" + RandomUtils.nextInt();
+    String outputTopic = "output-topic-" + RandomUtils.nextInt();
 
     Consumer<String, String> getKafkaConsumer() {
         return new KafkaConsumer<>(new Properties());
@@ -43,7 +48,7 @@ Producer<String, String> getKafkaProducer() {
 
     @SuppressWarnings("UnqualifiedFieldAccess")
     void run() {
-        this.parallelConsumer = setupConsumer();
+        this.parallelConsumer = setupParallelConsumer();
 
         // tag::example[]
         parallelConsumer.poll(record ->
@@ -53,40 +58,47 @@ void run() {
     }
 
     @SuppressWarnings({"FeatureEnvy", "MagicNumber"})
-    ParallelStreamProcessor<String, String> setupConsumer() {
+    ParallelStreamProcessor<String, String> setupParallelConsumer() {
         // tag::exampleSetup[]
-        var options = ParallelConsumerOptions.builder()
-                .ordering(KEY) // <1>
-                .maxConcurrency(1000) // <2>
-                .maxUncommittedMessagesToHandlePerPartition(10000) // <3>
-                .build();
+        ParallelConsumerOptions options = getOptions();
 
         Consumer<String, String> kafkaConsumer = getKafkaConsumer(); // <4>
+        Producer<String, String> kafkaProducer = getKafkaProducer();
+
+        ParallelStreamProcessor<String, String> eosStreamProcessor = ParallelStreamProcessor.createEosStreamProcessor(kafkaConsumer, kafkaProducer, options);
         if (!(kafkaConsumer instanceof MockConsumer)) {
-            kafkaConsumer.subscribe(UniLists.of(inputTopic)); // <5>
+            eosStreamProcessor.subscribe(UniLists.of(inputTopic)); // <5>
         }
 
-        return ParallelStreamProcessor.createEosStreamProcessor(kafkaConsumer, getKafkaProducer(), options);
+        return eosStreamProcessor;
         // end::exampleSetup[]
     }
 
+    ParallelConsumerOptions getOptions() {
+        var options = ParallelConsumerOptions.builder()
+                .ordering(KEY) // <1>
+                .maxMessagesToQueue(1000) // <2>
+                .maxNumberMessagesBeyondBaseCommitOffset(1000) // <3>
+                .build();
+        return options;
+    }
+
     void close() {
         this.parallelConsumer.close();
     }
 
     void runPollAndProduce() {
-        this.parallelConsumer = setupConsumer();
+        this.parallelConsumer = setupParallelConsumer();
 
         // tag::exampleProduce[]
         this.parallelConsumer.pollAndProduce(record -> {
                     var result = processBrokerRecord(record);
-                    ProducerRecord<String, String> produceRecord =
-                            new ProducerRecord<>(outputTopic, "a-key", result.payload);
-                    return UniLists.of(produceRecord);
-                }, consumeProduceResult ->
-                        log.info("Message {} saved to broker at offset {}",
-                                consumeProduceResult.getOut(),
-                                consumeProduceResult.getMeta().offset())
+                    return new ProducerRecord<>(outputTopic, record.key(), result.payload);
+                }, consumeProduceResult -> {
+                    log.debug("Message {} saved to broker at offset {}",
+                            consumeProduceResult.getOut(),
+                            consumeProduceResult.getMeta().offset());
+                }
         );
         // end::exampleProduce[]
     }
diff --git a/parallel-consumer-examples/parallel-consumer-example-core/src/test/java/io/confluent/parallelconsumer/examples/core/CoreAppTest.java b/parallel-consumer-examples/parallel-consumer-example-core/src/test/java/io/confluent/parallelconsumer/examples/core/CoreAppTest.java
index fb3b0a65c..49f13cbc9 100644
--- a/parallel-consumer-examples/parallel-consumer-example-core/src/test/java/io/confluent/parallelconsumer/examples/core/CoreAppTest.java
+++ b/parallel-consumer-examples/parallel-consumer-example-core/src/test/java/io/confluent/parallelconsumer/examples/core/CoreAppTest.java
@@ -4,6 +4,7 @@
  * Copyright (C) 2020 Confluent, Inc.
  */
 
+import io.confluent.csid.utils.KafkaTestUtils;
 import io.confluent.csid.utils.LongPollingMockConsumer;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -13,7 +14,6 @@
 import org.apache.kafka.clients.producer.MockProducer;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.common.TopicPartition;
-import org.assertj.core.api.Assertions;
 import org.awaitility.Awaitility;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
@@ -22,28 +22,27 @@
 import java.time.Duration;
 import java.util.HashMap;
 
-import static io.confluent.csid.utils.KafkaTestUtils.DEFAULT_GROUP_METADATA;
+import static io.confluent.parallelconsumer.ParallelEoSStreamProcessorTestBase.DEFAULT_GROUP_METADATA;
 import static org.mockito.Mockito.when;
 
 @Slf4j
 public class CoreAppTest {
 
-    TopicPartition tp = new TopicPartition(CoreApp.inputTopic, 0);
-
     @SneakyThrows
     @Test
     public void test() {
         log.info("Test start");
         CoreAppUnderTest coreApp = new CoreAppUnderTest();
+        TopicPartition tp = new TopicPartition(coreApp.inputTopic, 0);
 
         coreApp.run();
 
-        coreApp.mockConsumer.addRecord(new ConsumerRecord(CoreApp.inputTopic, 0, 0, "a key 1", "a value"));
-        coreApp.mockConsumer.addRecord(new ConsumerRecord(CoreApp.inputTopic, 0, 1, "a key 2", "a value"));
-        coreApp.mockConsumer.addRecord(new ConsumerRecord(CoreApp.inputTopic, 0, 2, "a key 3", "a value"));
+        coreApp.mockConsumer.addRecord(new ConsumerRecord(coreApp.inputTopic, 0, 0, "a key 1", "a value"));
+        coreApp.mockConsumer.addRecord(new ConsumerRecord(coreApp.inputTopic, 0, 1, "a key 2", "a value"));
+        coreApp.mockConsumer.addRecord(new ConsumerRecord(coreApp.inputTopic, 0, 2, "a key 3", "a value"));
 
-        Awaitility.await().pollInterval(Duration.ofSeconds(1)).untilAsserted(()->{
-            Assertions.assertThat(coreApp.mockConsumer.position(tp)).isEqualTo(3);
+        Awaitility.await().pollInterval(Duration.ofSeconds(1)).untilAsserted(() -> {
+            KafkaTestUtils.assertLastCommitIs(coreApp.mockConsumer, 3);
         });
 
         coreApp.close();
@@ -55,15 +54,16 @@ public void test() {
     public void testPollAndProduce() {
         log.info("Test start");
         CoreAppUnderTest coreApp = new CoreAppUnderTest();
+        TopicPartition tp = new TopicPartition(coreApp.inputTopic, 0);
 
         coreApp.runPollAndProduce();
 
-        coreApp.mockConsumer.addRecord(new ConsumerRecord(CoreApp.inputTopic, 0, 0, "a key 1", "a value"));
-        coreApp.mockConsumer.addRecord(new ConsumerRecord(CoreApp.inputTopic, 0, 1, "a key 2", "a value"));
-        coreApp.mockConsumer.addRecord(new ConsumerRecord(CoreApp.inputTopic, 0, 2, "a key 3", "a value"));
+        coreApp.mockConsumer.addRecord(new ConsumerRecord(coreApp.inputTopic, 0, 0, "a key 1", "a value"));
+        coreApp.mockConsumer.addRecord(new ConsumerRecord(coreApp.inputTopic, 0, 1, "a key 2", "a value"));
+        coreApp.mockConsumer.addRecord(new ConsumerRecord(coreApp.inputTopic, 0, 2, "a key 3", "a value"));
 
-        Awaitility.await().pollInterval(Duration.ofSeconds(1)).untilAsserted(()->{
-            Assertions.assertThat(coreApp.mockConsumer.position(tp)).isEqualTo(3);
+        Awaitility.await().pollInterval(Duration.ofSeconds(1)).untilAsserted(() -> {
+            KafkaTestUtils.assertLastCommitIs(coreApp.mockConsumer, 3);
         });
 
         coreApp.close();
@@ -72,6 +72,7 @@ public void testPollAndProduce() {
     class CoreAppUnderTest extends CoreApp {
 
         LongPollingMockConsumer<String, String> mockConsumer = Mockito.spy(new LongPollingMockConsumer<>(OffsetResetStrategy.EARLIEST));
+        TopicPartition tp = new TopicPartition(inputTopic, 0);
 
         @Override
         Consumer<String, String> getKafkaConsumer() {
@@ -85,7 +86,7 @@ Consumer<String, String> getKafkaConsumer() {
 
         @Override
         Producer<String, String> getKafkaProducer() {
-            return new MockProducer<>();
+            return new MockProducer<>(true, null, null);
         }
 
     }
diff --git a/parallel-consumer-examples/parallel-consumer-example-core/src/test/resources/logback-test.xml b/parallel-consumer-examples/parallel-consumer-example-core/src/test/resources/logback-temp-test.xml
similarity index 94%
rename from parallel-consumer-examples/parallel-consumer-example-core/src/test/resources/logback-test.xml
rename to parallel-consumer-examples/parallel-consumer-example-core/src/test/resources/logback-temp-test.xml
index 1d977eeab..aab528624 100644
--- a/parallel-consumer-examples/parallel-consumer-example-core/src/test/resources/logback-test.xml
+++ b/parallel-consumer-examples/parallel-consumer-example-core/src/test/resources/logback-temp-test.xml
@@ -16,7 +16,7 @@
     </root>
 
     <logger name="io.confluent.csid" level="info" />
-<!--    <logger name="io.confluent.csid" level="debug" />-->
+    <logger name="io.confluent.parallelconsumer" level="debug" />
 <!--    <logger name="io.confluent.csid" level="trace" />-->
 
     <!--    <logger name="io.confluent.csid.asyncconsumer.WorkManager" level="info" />-->
diff --git a/parallel-consumer-examples/parallel-consumer-example-streams/src/main/java/io/confluent/parallelconsumer/examples/streams/StreamsApp.java b/parallel-consumer-examples/parallel-consumer-example-streams/src/main/java/io/confluent/parallelconsumer/examples/streams/StreamsApp.java
index d85148239..1c192f5f9 100644
--- a/parallel-consumer-examples/parallel-consumer-example-streams/src/main/java/io/confluent/parallelconsumer/examples/streams/StreamsApp.java
+++ b/parallel-consumer-examples/parallel-consumer-example-streams/src/main/java/io/confluent/parallelconsumer/examples/streams/StreamsApp.java
@@ -5,7 +5,6 @@
  */
 
 
-import io.confluent.parallelconsumer.ParallelEoSStreamProcessor;
 import io.confluent.parallelconsumer.ParallelConsumerOptions;
 import io.confluent.parallelconsumer.ParallelStreamProcessor;
 import lombok.extern.slf4j.Slf4j;
@@ -69,7 +68,7 @@ void startStreams(Topology topology) {
     }
 
     void concurrentProcess() {
-        setupConsumer();
+        setupParallelConsumer();
 
         parallelConsumer.poll(record -> {
             log.info("Concurrently processing a record: {}", record);
@@ -78,16 +77,18 @@ void concurrentProcess() {
     }
     // end::example[]
 
-    private void setupConsumer() {
+    private void setupParallelConsumer() {
         var options = ParallelConsumerOptions.builder()
                 .ordering(ParallelConsumerOptions.ProcessingOrder.KEY)
                 .maxConcurrency(1000)
-                .maxUncommittedMessagesToHandlePerPartition(10000)
+                .maxUncommittedMessagesToHandle(10000)
                 .build();
 
         Consumer<String, String> kafkaConsumer = getKafkaConsumer();
 
-        parallelConsumer = ParallelStreamProcessor.createEosStreamProcessor(kafkaConsumer, getKafkaProducer(), options);
+        Producer<String, String> kafkaProducer = getKafkaProducer();
+        
+        parallelConsumer = ParallelStreamProcessor.createEosStreamProcessor(kafkaConsumer, kafkaProducer, options);
         parallelConsumer.subscribe(UniLists.of(outputTopicName));
     }
 
diff --git a/parallel-consumer-examples/parallel-consumer-example-streams/src/test/java/io/confluent/parallelconsumer/examples/streams/StreamsAppTest.java b/parallel-consumer-examples/parallel-consumer-example-streams/src/test/java/io/confluent/parallelconsumer/examples/streams/StreamsAppTest.java
index d8cc84425..a0e5a5acb 100644
--- a/parallel-consumer-examples/parallel-consumer-example-streams/src/test/java/io/confluent/parallelconsumer/examples/streams/StreamsAppTest.java
+++ b/parallel-consumer-examples/parallel-consumer-example-streams/src/test/java/io/confluent/parallelconsumer/examples/streams/StreamsAppTest.java
@@ -52,7 +52,7 @@ Consumer<String, String> getKafkaConsumer() {
 
         @Override
         Producer<String, String> getKafkaProducer() {
-            return kcu.createNewProducer(true);
+            return kcu.createNewProducer(false);
         }
 
         @Override
diff --git a/parallel-consumer-examples/parallel-consumer-example-vertx/pom.xml b/parallel-consumer-examples/parallel-consumer-example-vertx/pom.xml
index d21972d44..1af96390a 100644
--- a/parallel-consumer-examples/parallel-consumer-example-vertx/pom.xml
+++ b/parallel-consumer-examples/parallel-consumer-example-vertx/pom.xml
@@ -31,6 +31,13 @@
             <classifier>tests</classifier>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>io.confluent.parallelconsumer</groupId>
+            <artifactId>parallel-consumer-vertx</artifactId>
+            <version>${project.version}</version>
+            <classifier>tests</classifier>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
@@ -40,6 +47,12 @@
             <artifactId>awaitility</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.github.tomakehurst</groupId>
+            <artifactId>wiremock</artifactId>
+            <version>2.27.2</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
\ No newline at end of file
diff --git a/parallel-consumer-examples/parallel-consumer-example-vertx/src/main/java/io/confluent/parallelconsumer/examples/vertx/VertxApp.java b/parallel-consumer-examples/parallel-consumer-example-vertx/src/main/java/io/confluent/parallelconsumer/examples/vertx/VertxApp.java
index 0b4c1d023..62dc5b191 100644
--- a/parallel-consumer-examples/parallel-consumer-example-vertx/src/main/java/io/confluent/parallelconsumer/examples/vertx/VertxApp.java
+++ b/parallel-consumer-examples/parallel-consumer-example-vertx/src/main/java/io/confluent/parallelconsumer/examples/vertx/VertxApp.java
@@ -40,7 +40,7 @@ void run() {
         var options = ParallelConsumerOptions.builder()
                 .ordering(ParallelConsumerOptions.ProcessingOrder.KEY)
                 .maxConcurrency(1000)
-                .maxUncommittedMessagesToHandlePerPartition(10000)
+                .maxUncommittedMessagesToHandle(10000)
                 .build();
 
         Consumer<String, String> kafkaConsumer = getKafkaConsumer();
@@ -49,20 +49,26 @@ void run() {
         this.parallelConsumer = JStreamVertxParallelStreamProcessor.createEosStreamProcessor(kafkaConsumer,
                 getKafkaProducer(), options);
 
+        int port = getPort();
+
         // tag::example[]
         var resultStream = parallelConsumer.vertxHttpReqInfoStream(record -> {
             log.info("Concurrently constructing and returning RequestInfo from record: {}", record);
             Map<String, String> params = UniMaps.of("recordKey", record.key(), "payload", record.value());
-            return new RequestInfo("localhost", "/api", params); // <1>
+            return new RequestInfo("localhost", port, "/api", params); // <1>
         });
         // end::example[]
 
-        resultStream.forEach(x->{
+        resultStream.forEach(x -> {
             log.info("From result stream: {}", x);
         });
 
     }
 
+    protected int getPort() {
+        return 8080;
+    }
+
     void setupSubscription(Consumer<String, String> kafkaConsumer) {
         kafkaConsumer.subscribe(UniLists.of(inputTopic));
     }
diff --git a/parallel-consumer-examples/parallel-consumer-example-vertx/src/test/java/io/confluent/parallelconsumer/examples/vertx/VertxAppTest.java b/parallel-consumer-examples/parallel-consumer-example-vertx/src/test/java/io/confluent/parallelconsumer/examples/vertx/VertxAppTest.java
index 7652ad1d3..88cc778ee 100644
--- a/parallel-consumer-examples/parallel-consumer-example-vertx/src/test/java/io/confluent/parallelconsumer/examples/vertx/VertxAppTest.java
+++ b/parallel-consumer-examples/parallel-consumer-example-vertx/src/test/java/io/confluent/parallelconsumer/examples/vertx/VertxAppTest.java
@@ -4,7 +4,10 @@
  * Copyright (C) 2020 Confluent, Inc.
  */
 
+import io.confluent.csid.utils.KafkaTestUtils;
 import io.confluent.csid.utils.LongPollingMockConsumer;
+import io.confluent.parallelconsumer.vertx.VertxTest;
+import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.kafka.clients.consumer.Consumer;
@@ -13,7 +16,6 @@
 import org.apache.kafka.clients.producer.MockProducer;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.common.TopicPartition;
-import org.assertj.core.api.Assertions;
 import org.awaitility.Awaitility;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
@@ -22,9 +24,8 @@
 
 import java.time.Duration;
 import java.util.HashMap;
-import java.util.concurrent.TimeoutException;
 
-import static io.confluent.csid.utils.KafkaTestUtils.DEFAULT_GROUP_METADATA;
+import static io.confluent.parallelconsumer.ParallelEoSStreamProcessorTestBase.DEFAULT_GROUP_METADATA;
 
 @Slf4j
 public class VertxAppTest {
@@ -36,7 +37,10 @@ public class VertxAppTest {
     @Test
     public void test() {
         log.info("Test start");
-        VertxAppAppUnderTest coreApp = new VertxAppAppUnderTest();
+        VertxTest.setupWireMock();
+        int port = VertxTest.stubServer.port();
+
+        VertxAppAppUnderTest coreApp = new VertxAppAppUnderTest(port);
 
         coreApp.run();
 
@@ -45,17 +49,17 @@ public void test() {
         coreApp.mockConsumer.addRecord(new ConsumerRecord(VertxApp.inputTopic, 0, 2, "a key 3", "a value"));
 
         Awaitility.await().pollInterval(Duration.ofSeconds(1)).untilAsserted(()->{
-            Assertions.assertThat(coreApp.mockConsumer.position(tp)).isEqualTo(3);
+            KafkaTestUtils.assertLastCommitIs(coreApp.mockConsumer, 3);
         });
 
-        Assertions.assertThatExceptionOfType(TimeoutException.class)
-                .as("no server to receive request, should timeout trying to close. Could also setup wire mock...")
-                .isThrownBy(coreApp::close)
-                .withMessageContainingAll("Waiting", "records", "flight");
+        coreApp.close();
     }
 
+    @RequiredArgsConstructor
     class VertxAppAppUnderTest extends VertxApp {
 
+        private final int port;
+
         LongPollingMockConsumer<String, String> mockConsumer = Mockito.spy(new LongPollingMockConsumer<>(OffsetResetStrategy.EARLIEST));
 
         @Override
@@ -69,12 +73,17 @@ Consumer<String, String> getKafkaConsumer() {
 
         @Override
         Producer<String, String> getKafkaProducer() {
-            return new MockProducer<>();
+            return new MockProducer<>(true, null, null);
         }
 
         @Override
         void setupSubscription(Consumer<String, String> kafkaConsumer) {
             mockConsumer.assign(UniLists.of(tp));
         }
+
+        @Override
+        protected int getPort() {
+            return port;
+        }
     }
 }
diff --git a/parallel-consumer-vertx/src/main/java/io/confluent/parallelconsumer/vertx/JStreamVertxParallelEoSStreamProcessor.java b/parallel-consumer-vertx/src/main/java/io/confluent/parallelconsumer/vertx/JStreamVertxParallelEoSStreamProcessor.java
index ae2eae61b..2724cedae 100644
--- a/parallel-consumer-vertx/src/main/java/io/confluent/parallelconsumer/vertx/JStreamVertxParallelEoSStreamProcessor.java
+++ b/parallel-consumer-vertx/src/main/java/io/confluent/parallelconsumer/vertx/JStreamVertxParallelEoSStreamProcessor.java
@@ -25,6 +25,8 @@
 import java.util.function.Function;
 import java.util.stream.Stream;
 
+import static io.confluent.parallelconsumer.UserFunctions.carefullyRun;
+
 @Slf4j
 public class JStreamVertxParallelEoSStreamProcessor<K, V> extends VertxParallelEoSStreamProcessor<K, V>
         implements JStreamVertxParallelStreamProcessor<K, V> {
@@ -72,7 +74,7 @@ public Stream<VertxCPResult<K, V>> vertxHttpReqInfoStream(Function<ConsumerRecor
 
         Function<ConsumerRecord<K, V>, RequestInfo> requestInfoFunctionWrapped = x -> {
             result.in(x);
-            RequestInfo apply = requestInfoFunction.apply(x);
+            RequestInfo apply = carefullyRun(requestInfoFunction, x);
             result.requestInfo(Optional.of(apply));
             return apply;
         };
@@ -97,7 +99,8 @@ public Stream<VertxCPResult<K, V>> vertxHttpRequestStream(BiFunction<WebClient,
 
         BiFunction<WebClient, ConsumerRecord<K, V>, HttpRequest<Buffer>> requestInfoFunctionWrapped = (wc, x) -> {
             result.in(x);
-            HttpRequest<Buffer> apply = webClientRequestFunction.apply(wc, x);
+            HttpRequest<Buffer> apply = carefullyRun(webClientRequestFunction, wc, x);
+
             result.httpReq(Optional.of(apply));
             return apply;
         };
@@ -123,7 +126,8 @@ public Stream<VertxCPResult<K, V>> vertxHttpWebClientStream(
         BiFunction<WebClient, ConsumerRecord<K, V>, Future<HttpResponse<Buffer>>> wrappedFunc = (x, y) -> {
             // capture
             result.in(y);
-            Future<HttpResponse<Buffer>> apply = webClientRequestFunction.apply(x, y);
+            Future<HttpResponse<Buffer>> apply = carefullyRun(webClientRequestFunction, x, y);
+
             result.asr(apply);
             return apply;
         };
diff --git a/parallel-consumer-vertx/src/main/java/io/confluent/parallelconsumer/vertx/VertxParallelEoSStreamProcessor.java b/parallel-consumer-vertx/src/main/java/io/confluent/parallelconsumer/vertx/VertxParallelEoSStreamProcessor.java
index 68eefbb06..070be97df 100644
--- a/parallel-consumer-vertx/src/main/java/io/confluent/parallelconsumer/vertx/VertxParallelEoSStreamProcessor.java
+++ b/parallel-consumer-vertx/src/main/java/io/confluent/parallelconsumer/vertx/VertxParallelEoSStreamProcessor.java
@@ -4,9 +4,7 @@
  * Copyright (C) 2020 Confluent, Inc.
  */
 
-import io.confluent.parallelconsumer.ParallelEoSStreamProcessor;
-import io.confluent.parallelconsumer.ParallelConsumerOptions;
-import io.confluent.parallelconsumer.WorkContainer;
+import io.confluent.parallelconsumer.*;
 import io.vertx.core.AsyncResult;
 import io.vertx.core.Future;
 import io.vertx.core.Vertx;
@@ -34,6 +32,8 @@
 import java.util.function.Consumer;
 import java.util.function.Function;
 
+import static io.confluent.parallelconsumer.UserFunctions.carefullyRun;
+
 @Slf4j
 public class VertxParallelEoSStreamProcessor<K, V> extends ParallelEoSStreamProcessor<K, V>
         implements VertxParallelStreamProcessor<K, V> {
@@ -90,7 +90,8 @@ public void vertxHttpReqInfo(Function<ConsumerRecord<K, V>, RequestInfo> request
                                  Consumer<Future<HttpResponse<Buffer>>> onSend,
                                  Consumer<AsyncResult<HttpResponse<Buffer>>> onWebRequestComplete) {
         vertxHttpRequest((WebClient wc, ConsumerRecord<K, V> rec) -> {
-            RequestInfo reqInf = requestInfoFunction.apply(rec);
+            RequestInfo reqInf = carefullyRun(requestInfoFunction, rec);
+
             HttpRequest<Buffer> req = wc.get(reqInf.getPort(), reqInf.getHost(), reqInf.getContextPath());
             Map<String, String> params = reqInf.getParams();
             for (var entry : params.entrySet()) {
@@ -106,7 +107,7 @@ public void vertxHttpRequest(BiFunction<WebClient, ConsumerRecord<K, V>, HttpReq
                                  Consumer<AsyncResult<HttpResponse<Buffer>>> onWebRequestComplete) { // TODO remove, redundant over onSend?
 
         vertxHttpWebClient((webClient, record) -> {
-            HttpRequest<Buffer> call = webClientRequestFunction.apply(webClient, record);
+            HttpRequest<Buffer> call = carefullyRun(webClientRequestFunction, webClient, record);
 
             Future<HttpResponse<Buffer>> send = call.send(); // dispatches the work to vertx
 
@@ -124,7 +125,8 @@ public void vertxHttpWebClient(BiFunction<WebClient, ConsumerRecord<K, V>, Futur
                                    Consumer<Future<HttpResponse<Buffer>>> onSend) {
 
         Function<ConsumerRecord<K, V>, List<Future<HttpResponse<Buffer>>>> userFuncWrapper = (record) -> {
-            Future<HttpResponse<Buffer>> send = webClientRequestFunction.apply(webClient, record);
+
+            Future<HttpResponse<Buffer>> send = carefullyRun(webClientRequestFunction, webClient, record);
 
             // send callback
             onSend.accept(send);
@@ -232,7 +234,7 @@ private boolean isVertxWork(List<?> resultsFromUserFunction) {
     @Override
     public void close(Duration timeout, DrainingMode drainMode) {
         log.info("Vert.x async consumer closing...");
-        waitForNoInFlight(timeout);
+        waitForProcessedNotCommitted(timeout);
         super.close(timeout, drainMode);
         webClient.close();
         Future<Void> close = vertx.close();
diff --git a/parallel-consumer-vertx/src/test/java/io/confluent/parallelconsumer/vertx/VertxTest.java b/parallel-consumer-vertx/src/test/java/io/confluent/parallelconsumer/vertx/VertxTest.java
index 22b634f02..58ff082a8 100644
--- a/parallel-consumer-vertx/src/test/java/io/confluent/parallelconsumer/vertx/VertxTest.java
+++ b/parallel-consumer-vertx/src/test/java/io/confluent/parallelconsumer/vertx/VertxTest.java
@@ -41,6 +41,7 @@
 
 import static com.github.tomakehurst.wiremock.client.WireMock.*;
 import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.TRANSACTIONAL_PRODUCER;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Awaitility.await;
@@ -52,7 +53,7 @@ public class VertxTest extends ParallelEoSStreamProcessorTestBase {
 
     JStreamVertxParallelEoSStreamProcessor<String, String> vertxAsync;
 
-    protected static WireMockServer stubServer;
+    public static WireMockServer stubServer;
 
     protected static final String stubResponse = "Good times.";
 
@@ -68,12 +69,13 @@ VertxParallelEoSStreamProcessor.RequestInfo getBadHost() {
     public static void setupWireMock() {
         WireMockConfiguration options = wireMockConfig().dynamicPort();
         stubServer = new WireMockServer(options);
-        stubServer.start();
         MappingBuilder mappingBuilder = get(urlPathEqualTo("/"))
                 .willReturn(aResponse()
                         .withBody(stubResponse));
-        stubServer.stubFor(
-                mappingBuilder);
+        stubServer.stubFor(mappingBuilder);
+        stubServer.stubFor(get(urlPathEqualTo("/api")).
+                willReturn(aResponse().withBody(stubResponse)));
+        stubServer.start();
     }
 
     @Override
@@ -81,7 +83,7 @@ protected ParallelEoSStreamProcessor initAsyncConsumer(ParallelConsumerOptions p
         VertxOptions vertxOptions = new VertxOptions();
         Vertx vertx = Vertx.vertx(vertxOptions);
         WebClient wc = WebClient.create(vertx);
-        ParallelConsumerOptions build = ParallelConsumerOptions.builder().build();
+        ParallelConsumerOptions build = ParallelConsumerOptions.builder().commitMode(TRANSACTIONAL_PRODUCER).usingTransactionalProducer(true).build();
         vertxAsync = new JStreamVertxParallelEoSStreamProcessor<>(consumerSpy, producerSpy, vertx, wc, build);
 
         return vertxAsync;
diff --git a/src/docs/README.adoc b/src/docs/README.adoc
index ddbe57429..40f8ee4b8 100644
--- a/src/docs/README.adoc
+++ b/src/docs/README.adoc
@@ -477,16 +477,33 @@ When your function is actually run, a result object will be streamed back to you
 After your operation completes, you can also choose to publish a result message back to Kafka.
 The message publishing metadata can be streamed back to your client code.
 
-== Apache Kafka EoS Transaction Model
+== Commit Mode
 
-The system uses Kafka's Exactly Once Semantics (EoS) system.
+The system gives you three choices for how to do offset commits.
+The simplest of the three are the two Consummer commits modes.
+They are of course, `sycnhronous` and `asynchronous` mode.
+The `transactional` mode is explained in the next section.
+
+`Asychornous` mode is faster, as it doesn't block the control loop.
+
+`Sycnhronous` will block the processing loop until a successful commit response is received, however, `asycnhronous` will still be capped by the max processing settings in the `Options` class.
+
+If you're used to using the auto commit mode in the normal Kafka consumer, you can think of the `Asychornous` mode being similar to this.
+We suggest starting with this mode, and it is the default.
+
+=== Apache Kafka EoS Transaction Model
+
+There is also the option to use Kafka's Exactly Once Semantics (EoS) system.
 This causes all messages produced as a result of processing a message to be committed within a transaction, along with their source offset.
 This means that even under failure, the results will exist exactly once in the Kafka output topic.
-If as a part of
-your processing, you create side effects in other systems, this pertains to the usual idempotency requirements when breaking of EoS Kafka boundaries.
+If as a part of your processing, you create side effects in other systems, this pertains to the usual idempotency requirements when breaking of EoS Kafka boundaries.
+
+NOTE:: As with the `synchronous` processing mode, this will also block the processing loop until a successful transaction completes
 
 CAUTION: This cannot be true for any externally integrated third party system, unless that system is __idempotent__.
 
+For implementations details, see the <<Transactional System Architecture>> section.
+
 [[streams-usage]]
 == Using with Kafka Streams
 
@@ -673,6 +690,10 @@ Instead of the work thread pool count being the degree of concurrency, it is con
 .Vert.x Architecture
 image::https://lucid.app/publicSegments/view/509df410-5997-46be-98e7-ac7f241780b4/image.png[Vert.x Architecture, align="center"]
 
+=== Transactional System Architecture
+
+image::https://lucid.app/publicSegments/view/7480d948-ed7d-4370-a308-8ec12e6b453b/image.png[]
+
 === Offset Map
 
 ==== Storage