diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 000000000..7dda17caa
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,285 @@
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = false
+max_line_length = 120
+tab_width = 2
+ij_continuation_indent_size = 8
+ij_formatter_off_tag = @formatter:off
+ij_formatter_on_tag = @formatter:on
+ij_formatter_tags_enabled = false
+ij_smart_tabs = false
+ij_visual_guides = none
+ij_wrap_on_typing = false
+
+[*.java]
+indent_size = 4
+tab_width = 4
+ij_java_align_consecutive_assignments = false
+ij_java_align_consecutive_variable_declarations = false
+ij_java_align_group_field_declarations = false
+ij_java_align_multiline_annotation_parameters = false
+ij_java_align_multiline_array_initializer_expression = false
+ij_java_align_multiline_assignment = false
+ij_java_align_multiline_binary_operation = false
+ij_java_align_multiline_chained_methods = false
+ij_java_align_multiline_extends_list = false
+ij_java_align_multiline_for = true
+ij_java_align_multiline_method_parentheses = false
+ij_java_align_multiline_parameters = true
+ij_java_align_multiline_parameters_in_calls = false
+ij_java_align_multiline_parenthesized_expression = false
+ij_java_align_multiline_records = true
+ij_java_align_multiline_resources = true
+ij_java_align_multiline_ternary_operation = false
+ij_java_align_multiline_text_blocks = false
+ij_java_align_multiline_throws_list = false
+ij_java_align_subsequent_simple_methods = false
+ij_java_align_throws_keyword = false
+ij_java_annotation_parameter_wrap = off
+ij_java_array_initializer_new_line_after_left_brace = false
+ij_java_array_initializer_right_brace_on_new_line = false
+ij_java_array_initializer_wrap = off
+ij_java_assert_statement_colon_on_next_line = false
+ij_java_assert_statement_wrap = off
+ij_java_assignment_wrap = off
+ij_java_binary_operation_sign_on_next_line = false
+ij_java_binary_operation_wrap = off
+ij_java_blank_lines_after_anonymous_class_header = 0
+ij_java_blank_lines_after_class_header = 0
+ij_java_blank_lines_after_imports = 1
+ij_java_blank_lines_after_package = 1
+ij_java_blank_lines_around_class = 1
+ij_java_blank_lines_around_field = 0
+ij_java_blank_lines_around_field_in_interface = 0
+ij_java_blank_lines_around_initializer = 1
+ij_java_blank_lines_around_method = 1
+ij_java_blank_lines_around_method_in_interface = 1
+ij_java_blank_lines_before_class_end = 0
+ij_java_blank_lines_before_imports = 1
+ij_java_blank_lines_before_method_body = 0
+ij_java_blank_lines_before_package = 0
+ij_java_block_brace_style = end_of_line
+ij_java_block_comment_at_first_column = true
+ij_java_builder_methods = none
+ij_java_call_parameters_new_line_after_left_paren = false
+ij_java_call_parameters_right_paren_on_new_line = false
+ij_java_call_parameters_wrap = off
+ij_java_case_statement_on_separate_line = true
+ij_java_catch_on_new_line = false
+ij_java_class_annotation_wrap = split_into_lines
+ij_java_class_brace_style = end_of_line
+ij_java_class_count_to_use_import_on_demand = 5
+ij_java_class_names_in_javadoc = 1
+ij_java_do_not_indent_top_level_class_members = false
+ij_java_do_not_wrap_after_single_annotation = false
+ij_java_do_while_brace_force = never
+ij_java_doc_add_blank_line_after_description = true
+ij_java_doc_add_blank_line_after_param_comments = false
+ij_java_doc_add_blank_line_after_return = false
+ij_java_doc_add_p_tag_on_empty_lines = true
+ij_java_doc_align_exception_comments = true
+ij_java_doc_align_param_comments = true
+ij_java_doc_do_not_wrap_if_one_line = false
+ij_java_doc_enable_formatting = true
+ij_java_doc_enable_leading_asterisks = true
+ij_java_doc_indent_on_continuation = true
+ij_java_doc_keep_empty_lines = true
+ij_java_doc_keep_empty_parameter_tag = true
+ij_java_doc_keep_empty_return_tag = true
+ij_java_doc_keep_empty_throws_tag = true
+ij_java_doc_keep_invalid_tags = true
+ij_java_doc_param_description_on_new_line = false
+ij_java_doc_preserve_line_breaks = false
+ij_java_doc_use_throws_not_exception_tag = true
+ij_java_else_on_new_line = false
+ij_java_enum_constants_wrap = off
+ij_java_extends_keyword_wrap = off
+ij_java_extends_list_wrap = off
+ij_java_field_annotation_wrap = split_into_lines
+ij_java_finally_on_new_line = false
+ij_java_for_brace_force = never
+ij_java_for_statement_new_line_after_left_paren = false
+ij_java_for_statement_right_paren_on_new_line = false
+ij_java_for_statement_wrap = off
+ij_java_generate_final_locals = true
+ij_java_generate_final_parameters = true
+ij_java_if_brace_force = never
+ij_java_imports_layout = *, |, javax.**, java.**, |, $*
+ij_java_indent_case_from_switch = true
+ij_java_insert_inner_class_imports = false
+ij_java_insert_override_annotation = true
+ij_java_keep_blank_lines_before_right_brace = 2
+ij_java_keep_blank_lines_between_package_declaration_and_header = 2
+ij_java_keep_blank_lines_in_code = 2
+ij_java_keep_blank_lines_in_declarations = 2
+ij_java_keep_builder_methods_indents = false
+ij_java_keep_control_statement_in_one_line = true
+ij_java_keep_first_column_comment = true
+ij_java_keep_indents_on_empty_lines = false
+ij_java_keep_line_breaks = true
+ij_java_keep_multiple_expressions_in_one_line = false
+ij_java_keep_simple_blocks_in_one_line = false
+ij_java_keep_simple_classes_in_one_line = false
+ij_java_keep_simple_lambdas_in_one_line = false
+ij_java_keep_simple_methods_in_one_line = false
+ij_java_label_indent_absolute = false
+ij_java_label_indent_size = 0
+ij_java_lambda_brace_style = end_of_line
+ij_java_layout_static_imports_separately = true
+ij_java_line_comment_add_space = false
+ij_java_line_comment_at_first_column = true
+ij_java_method_annotation_wrap = split_into_lines
+ij_java_method_brace_style = end_of_line
+ij_java_method_call_chain_wrap = off
+ij_java_method_parameters_new_line_after_left_paren = false
+ij_java_method_parameters_right_paren_on_new_line = false
+ij_java_method_parameters_wrap = off
+ij_java_modifier_list_wrap = false
+ij_java_names_count_to_use_import_on_demand = 3
+ij_java_new_line_after_lparen_in_record_header = false
+ij_java_packages_to_use_import_on_demand = java.awt.*, javax.swing.*
+ij_java_parameter_annotation_wrap = off
+ij_java_parentheses_expression_new_line_after_left_paren = false
+ij_java_parentheses_expression_right_paren_on_new_line = false
+ij_java_place_assignment_sign_on_next_line = false
+ij_java_prefer_longer_names = true
+ij_java_prefer_parameters_wrap = false
+ij_java_record_components_wrap = normal
+ij_java_repeat_synchronized = true
+ij_java_replace_instanceof_and_cast = false
+ij_java_replace_null_check = true
+ij_java_replace_sum_lambda_with_method_ref = true
+ij_java_resource_list_new_line_after_left_paren = false
+ij_java_resource_list_right_paren_on_new_line = false
+ij_java_resource_list_wrap = off
+ij_java_rparen_on_new_line_in_record_header = false
+ij_java_space_after_closing_angle_bracket_in_type_argument = false
+ij_java_space_after_colon = true
+ij_java_space_after_comma = true
+ij_java_space_after_comma_in_type_arguments = true
+ij_java_space_after_for_semicolon = true
+ij_java_space_after_quest = true
+ij_java_space_after_type_cast = true
+ij_java_space_before_annotation_array_initializer_left_brace = false
+ij_java_space_before_annotation_parameter_list = false
+ij_java_space_before_array_initializer_left_brace = false
+ij_java_space_before_catch_keyword = true
+ij_java_space_before_catch_left_brace = true
+ij_java_space_before_catch_parentheses = true
+ij_java_space_before_class_left_brace = true
+ij_java_space_before_colon = true
+ij_java_space_before_colon_in_foreach = true
+ij_java_space_before_comma = false
+ij_java_space_before_do_left_brace = true
+ij_java_space_before_else_keyword = true
+ij_java_space_before_else_left_brace = true
+ij_java_space_before_finally_keyword = true
+ij_java_space_before_finally_left_brace = true
+ij_java_space_before_for_left_brace = true
+ij_java_space_before_for_parentheses = true
+ij_java_space_before_for_semicolon = false
+ij_java_space_before_if_left_brace = true
+ij_java_space_before_if_parentheses = true
+ij_java_space_before_method_call_parentheses = false
+ij_java_space_before_method_left_brace = true
+ij_java_space_before_method_parentheses = false
+ij_java_space_before_opening_angle_bracket_in_type_parameter = false
+ij_java_space_before_quest = true
+ij_java_space_before_switch_left_brace = true
+ij_java_space_before_switch_parentheses = true
+ij_java_space_before_synchronized_left_brace = true
+ij_java_space_before_synchronized_parentheses = true
+ij_java_space_before_try_left_brace = true
+ij_java_space_before_try_parentheses = true
+ij_java_space_before_type_parameter_list = false
+ij_java_space_before_while_keyword = true
+ij_java_space_before_while_left_brace = true
+ij_java_space_before_while_parentheses = true
+ij_java_space_inside_one_line_enum_braces = false
+ij_java_space_within_empty_array_initializer_braces = false
+ij_java_space_within_empty_method_call_parentheses = false
+ij_java_space_within_empty_method_parentheses = false
+ij_java_spaces_around_additive_operators = true
+ij_java_spaces_around_assignment_operators = true
+ij_java_spaces_around_bitwise_operators = true
+ij_java_spaces_around_equality_operators = true
+ij_java_spaces_around_lambda_arrow = true
+ij_java_spaces_around_logical_operators = true
+ij_java_spaces_around_method_ref_dbl_colon = false
+ij_java_spaces_around_multiplicative_operators = true
+ij_java_spaces_around_relational_operators = true
+ij_java_spaces_around_shift_operators = true
+ij_java_spaces_around_type_bounds_in_type_parameters = true
+ij_java_spaces_around_unary_operator = false
+ij_java_spaces_within_angle_brackets = false
+ij_java_spaces_within_annotation_parentheses = false
+ij_java_spaces_within_array_initializer_braces = false
+ij_java_spaces_within_braces = false
+ij_java_spaces_within_brackets = false
+ij_java_spaces_within_cast_parentheses = false
+ij_java_spaces_within_catch_parentheses = false
+ij_java_spaces_within_for_parentheses = false
+ij_java_spaces_within_if_parentheses = false
+ij_java_spaces_within_method_call_parentheses = false
+ij_java_spaces_within_method_parentheses = false
+ij_java_spaces_within_parentheses = false
+ij_java_spaces_within_record_header = false
+ij_java_spaces_within_switch_parentheses = false
+ij_java_spaces_within_synchronized_parentheses = false
+ij_java_spaces_within_try_parentheses = false
+ij_java_spaces_within_while_parentheses = false
+ij_java_special_else_if_treatment = true
+ij_java_subclass_name_suffix = Impl
+ij_java_ternary_operation_signs_on_next_line = false
+ij_java_ternary_operation_wrap = off
+ij_java_test_name_suffix = Test
+ij_java_throws_keyword_wrap = off
+ij_java_throws_list_wrap = off
+ij_java_use_external_annotations = false
+ij_java_use_fq_class_names = false
+ij_java_use_relative_indents = false
+ij_java_use_single_class_imports = true
+ij_java_variable_annotation_wrap = off
+ij_java_visibility = public
+ij_java_while_brace_force = never
+ij_java_while_on_new_line = false
+ij_java_wrap_comments = true
+ij_java_wrap_first_method_in_call_chain = false
+ij_java_wrap_long_lines = false
+
+[.editorconfig]
+ij_editorconfig_align_group_field_declarations = false
+ij_editorconfig_space_after_colon = false
+ij_editorconfig_space_after_comma = true
+ij_editorconfig_space_before_colon = false
+ij_editorconfig_space_before_comma = false
+ij_editorconfig_spaces_around_assignment_operators = true
+
+[{*.ad,*.adoc,*.asciidoc,.asciidoctorconfig}]
+ij_asciidoc_blank_lines_after_header = 1
+ij_asciidoc_blank_lines_keep_after_header = 1
+ij_asciidoc_formatting_enabled = true
+ij_asciidoc_one_sentence_per_line = true
+
+[{*.pom,*.xml}]
+indent_size = 4
+tab_width = 4
+ij_xml_align_attributes = true
+ij_xml_align_text = false
+ij_xml_attribute_wrap = normal
+ij_xml_block_comment_at_first_column = true
+ij_xml_keep_blank_lines = 2
+ij_xml_keep_indents_on_empty_lines = false
+ij_xml_keep_line_breaks = true
+ij_xml_keep_line_breaks_in_text = false
+ij_xml_keep_whitespaces = false
+ij_xml_keep_whitespaces_around_cdata = preserve
+ij_xml_keep_whitespaces_inside_cdata = false
+ij_xml_line_comment_at_first_column = true
+ij_xml_space_after_tag_name = false
+ij_xml_space_around_equals_in_attribute = false
+ij_xml_space_inside_empty_tag = false
+ij_xml_text_wrap = off
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 000000000..b5b1656cf
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* astubbs
\ No newline at end of file
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..ac73c3ed4
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,14 @@
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+ - package-ecosystem: "maven" # See documentation for possible values
+ directory: "/" # Location of package manifests
+ schedule:
+ interval: "daily"
+# Don't use any github-actions anymore
+# - package-ecosystem: "github-actions"
+# directory: "/"
+# schedule:
+# interval: "daily"
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 000000000..803d20097
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,6 @@
+Description...
+
+### Checklist
+
+- [ ] Documentation (if applicable)
+- [ ] Changelog
\ No newline at end of file
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
new file mode 100644
index 000000000..0a4cde6a1
--- /dev/null
+++ b/.github/workflows/maven.yml
@@ -0,0 +1,113 @@
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+
+# Tests disabled due to flakiness with under resourced github test machines. Confluent Jira works fine. Will fix later.
+name: Unit tests only
+
+on:
+ push:
+ branches: [ master ]
+ pull_request:
+ branches: [ master ]
+
+jobs:
+ build:
+ strategy:
+ fail-fast: false
+ matrix:
+ # Why not? because we can.
+ # 2.0.1, 2.1.1, 2.2.2, 2.3.1, 2.4.1 don't work - needs zstd and some kafka client libs.
+ # Doesn't mean it couldn't be modified slightly to work...
+ #ak: [ 2.5.1, 2.6.1, 2.7.0, 2.8.1, 3.0.1, 3.1.0 ]
+ # 25 and 26 include a dep with a vulnerability which ossindex fails the build for
+ ak: [ 2.7.0, 2.8.1, 3.0.1, 3.1.0 ]
+ #ak: [ 2.7.0 ]
+ #jdk: [ '-P jvm8-release -Djvm8.location=/opt/hostedtoolcache/Java_Zulu_jdk/8.0.332-9/x64', '' ]
+ # TG currently targets 11, so can't run the tests on 8 https://github.com/astubbs/truth-generator/issues/114
+ jdk: [ '' ]
+ experimental: [ false ]
+ name: [ "Stable AK version" ]
+ include:
+ # AK 2.4 not supported
+ # - ak: "'[2.4.1,2.5)'" # currently failing
+ # experimental: true
+ # name: "Oldest AK breaking version 2.4.1+ (below 2.5.0) expected to fail"
+ - ak: "'[2.7.0,4)'" # currently failing
+ experimental: true
+ name: "Newest AK version 2.7.0+?"
+
+ continue-on-error: ${{ matrix.experimental }}
+ name: "AK: ${{ matrix.ak }} JDK: ${{ matrix.jdk }}"
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Setup JDK 1.8
+ uses: actions/setup-java@v3
+ with:
+ java-version: '8'
+ distribution: 'zulu'
+ cache: 'maven'
+
+ # the patch version will be upgraded silently causing the build to eventually start failing - need to store this as a var - possible?
+ - name: Show java 1.8 home
+ # /opt/hostedtoolcache/Java_Zulu_jdk/8.0.332-9/x64/bin/java
+ run: which java
+
+ # - name: Setup JDK 1.9
+ # uses: actions/setup-java@v1
+ # with:
+ # java-version: 1.9
+
+ # - name: Show java 1.9 home
+ # /opt/hostedtoolcache/jdk/9.0.7/x64
+ # run: which java
+
+ - name: Setup JDK 17
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'zulu'
+ java-version: '17'
+ cache: 'maven'
+
+ - name: Show java 17 home
+ # /opt/hostedtoolcache/jdk/13.0.2/x64/bin/java
+ run: which java
+
+ # - name: Show java version
+ # run: java -version
+
+ # - name: Show mvn version
+ # run: mvn -version
+
+ # - name: Build with Maven on Java 13
+ # run: mvn -B package --file pom.xml
+
+
+ # done automatically now
+ # - name: Cache Maven packages
+ # uses: actions/cache@v2.1.7
+ # with:
+ # path: ~/.m2/repository
+ # key: ${{ runner.os }}-m2
+ # restore-keys: ${{ runner.os }}-m2
+
+ - name: Test with Maven
+ run: mvn -Pci -B package ${{ matrix.jdk }} -Dkafka.version=${{ matrix.ak }} -Dlicense.skip
+
+# - name: Archive test results
+# if: ${{ always() }}
+# uses: actions/upload-artifact@v2
+# with:
+# name: test-reports
+# path: target/**-reports/*
+# retention-days: 14
+#
+# - name: Archive surefire test results
+# if: ${{ always() }}
+# uses: actions/upload-artifact@v2
+# with:
+# name: test-reports
+# path: target/surefire-reports/*
+# retention-days: 14
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000..48a79e79a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,75 @@
+.DS_Store
+
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+
+*.versionsBackup
+
+# JENV
+.java-version
+
+delombok/
+**/*.releaseBackup
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+.idea/sonarlint/
+.idea/libraries/
+
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn. Uncomment if using
+# auto-import.
+.idea/artifacts
+.idea/compiler.xml
+.idea/jarRepositories.xml
+.idea/modules.xml
+.idea/*.iml
+.idea/modules
+*.iml
+*.ipr
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# Maven
+target
+release.properties
+/.idea/encodings.xml
+/.idea/misc.xml
+/.idea/codeStyles/Project.xml
+/.idea/inspectionProfiles/Project_Default.xml
+/.idea/uiDesigner.xml
+/.idea/vcs.xml
diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml
new file mode 100644
index 000000000..a55e7a179
--- /dev/null
+++ b/.idea/codeStyles/codeStyleConfig.xml
@@ -0,0 +1,5 @@
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All.xml b/.idea/runConfigurations/All.xml
new file mode 100644
index 000000000..1080d2485
--- /dev/null
+++ b/.idea/runConfigurations/All.xml
@@ -0,0 +1,26 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_Core.xml b/.idea/runConfigurations/All_Core.xml
new file mode 100644
index 000000000..012d635b1
--- /dev/null
+++ b/.idea/runConfigurations/All_Core.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_Core_Unit_Tests.xml b/.idea/runConfigurations/All_Core_Unit_Tests.xml
new file mode 100644
index 000000000..6e0c4a4e6
--- /dev/null
+++ b/.idea/runConfigurations/All_Core_Unit_Tests.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_Modules.xml b/.idea/runConfigurations/All_Modules.xml
new file mode 100644
index 000000000..ab45cfb23
--- /dev/null
+++ b/.idea/runConfigurations/All_Modules.xml
@@ -0,0 +1,9 @@
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_Modules___Unit_Tests.xml b/.idea/runConfigurations/All_Modules___Unit_Tests.xml
new file mode 100644
index 000000000..eee0435ec
--- /dev/null
+++ b/.idea/runConfigurations/All_Modules___Unit_Tests.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_Reactor_Unit_Tests.xml b/.idea/runConfigurations/All_Reactor_Unit_Tests.xml
new file mode 100644
index 000000000..b90481929
--- /dev/null
+++ b/.idea/runConfigurations/All_Reactor_Unit_Tests.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_Vertx.xml b/.idea/runConfigurations/All_Vertx.xml
new file mode 100644
index 000000000..a89f6b4b2
--- /dev/null
+++ b/.idea/runConfigurations/All_Vertx.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_Vertx_Unit_Tests.xml b/.idea/runConfigurations/All_Vertx_Unit_Tests.xml
new file mode 100644
index 000000000..0b46a963a
--- /dev/null
+++ b/.idea/runConfigurations/All_Vertx_Unit_Tests.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_example_core.xml b/.idea/runConfigurations/All_example_core.xml
new file mode 100644
index 000000000..9d53c6d3c
--- /dev/null
+++ b/.idea/runConfigurations/All_example_core.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_example_vertx.xml b/.idea/runConfigurations/All_example_vertx.xml
new file mode 100644
index 000000000..b0e464ccf
--- /dev/null
+++ b/.idea/runConfigurations/All_example_vertx.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/All_examples.xml b/.idea/runConfigurations/All_examples.xml
new file mode 100644
index 000000000..98c78936f
--- /dev/null
+++ b/.idea/runConfigurations/All_examples.xml
@@ -0,0 +1,22 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/README.xml b/.idea/runConfigurations/README.xml
new file mode 100644
index 000000000..cb6986814
--- /dev/null
+++ b/.idea/runConfigurations/README.xml
@@ -0,0 +1,27 @@
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/_Tag__transactions__.xml b/.idea/runConfigurations/_Tag__transactions__.xml
new file mode 100644
index 000000000..26e7f9dc9
--- /dev/null
+++ b/.idea/runConfigurations/_Tag__transactions__.xml
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/_release_prepare_.xml b/.idea/runConfigurations/_release_prepare_.xml
new file mode 100644
index 000000000..adaacd3ac
--- /dev/null
+++ b/.idea/runConfigurations/_release_prepare_.xml
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/asciidoc_template_build.xml b/.idea/runConfigurations/asciidoc_template_build.xml
new file mode 100644
index 000000000..b5ef11263
--- /dev/null
+++ b/.idea/runConfigurations/asciidoc_template_build.xml
@@ -0,0 +1,28 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/generate_test_sources__e.xml b/.idea/runConfigurations/generate_test_sources__e.xml
new file mode 100644
index 000000000..2456e1d82
--- /dev/null
+++ b/.idea/runConfigurations/generate_test_sources__e.xml
@@ -0,0 +1,30 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/license_format.xml b/.idea/runConfigurations/license_format.xml
new file mode 100644
index 000000000..09f2bc9ab
--- /dev/null
+++ b/.idea/runConfigurations/license_format.xml
@@ -0,0 +1,27 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/truth_generate__pc_.xml b/.idea/runConfigurations/truth_generate__pc_.xml
new file mode 100644
index 000000000..8778f1801
--- /dev/null
+++ b/.idea/runConfigurations/truth_generate__pc_.xml
@@ -0,0 +1,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties
new file mode 100644
index 000000000..08ea486aa
--- /dev/null
+++ b/.mvn/wrapper/maven-wrapper.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.0/apache-maven-3.9.0-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar
diff --git a/.travis-archived.yml b/.travis-archived.yml
new file mode 100644
index 000000000..eb04f839f
--- /dev/null
+++ b/.travis-archived.yml
@@ -0,0 +1,60 @@
+#
+# Copyright (C) 2020-2022 Confluent, Inc.
+#
+
+# Archived in favour of github actions
+
+language: java
+jdk:
+ - openjdk13
+
+sudo: required
+
+# docker and docker in docker setup. disabled for faster builds until needed
+#services:
+# - docker
+#
+#before_cache:
+# - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
+# - rm -fr $HOME/.gradle/caches/*/plugin-resolution/
+#cache:
+# directories:
+# - "$HOME/.gradle/caches/"
+# - "$HOME/.gradle/wrapper/"
+#
+#before_install:
+# - sudo rm /usr/local/bin/docker-compose
+# - curl -L https://github.com/docker/compose/releases/download/1.24.1/docker-compose-Linux-x86_64 > docker-compose
+# - chmod +x docker-compose
+# - sudo mv docker-compose /usr/local/bin
+
+cache:
+ directories:
+ - "$HOME/.m2"
+
+#before_install:
+# - $TRAVIS_BUILD_DIR/install-jdk.sh --install openjdk9 --target ~/openjdk9
+# - ls -la ~/
+
+install: skip
+
+# Removed for https://about.codecov.io/security-update/
+# APRIL 15TH, 2021
+# Bash Uploader Security Update
+# after_success:
+# - bash <(curl -s https://codecov.io/bash)
+
+addons:
+ sonarcloud:
+ organization: "astubbs"
+ token:
+ secure: "zUbcZgSuBEi1j8nboM6y5Eoj7Go6OdcW4h9IYk7iYbHVlSwTCzq1Tez0FG2moSvyxgwaXD+ySd4XvbH1hT04R59b2fumFt2eWO3FbSHFrE3dXdOxliz47FLXkqpg8MvEWNkF4hPHwsi9LTXl4u7UuFRTqOCLilA5RUBZyzQ03AExMQJdZgdlestarlys40thISEGHNmNd4nr+EEekkaekN+1iE3v4HZpXXv8COLdp10Hehl6RPg9ooCZ3g8B++IOI5MdRxMf3HeERyKapMN1xGT6ZpeCkaFd/GbbAjzjlhKHIJ37Mmo2l9nJs/9dpBml62SFb1WdpG+7610e49vQbHuy1yb9h1XOJPdw45AZw+g61/6LTmGuNNlkUTstELQEN0iDoo0GqEMtIlVplKUcnzVAXtldvKU4Ph7Satdk4wdA3K+4E+zhaMaJhzUBNluChF5JldOOQDV5odt9K0rZCH/4zGbmsR0nev+g6JW1DX96laqLkuA4Am1aDistZSjt3T3HqhAXcpf/8VW4p1HKtYhsmMybnOuqoOH7sBpwxXoR0Myvj6FMrh4On/t+/vYQSJm+vyiyLShP/Bouk+azygIcjG3ZUmaFpumRpKesK6C9EYCA2d59vnsYU0/Ob+IwvrDVY40HQl41A+ooC+WVQf8scvvFAIJpYH6V/0TCWXg="
+
+script:
+ # the following command line builds the project, runs the tests with coverage and then execute the SonarCloud analysis
+ # - mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent install sonar:sonar versions:display-dependency-updates
+ # -Djvm8.location=/usr/lib/jvm/java-8-openjdk-amd64/jre
+ # -Djvm9.location=/home/travis/openjdk9
+ - ~/bin/install-jdk.sh --target /home/travis/openjdk9 --feature 9
+ - mvn -version
+ - mvn -Pci clean verify versions:display-plugin-updates versions:display-property-updates versions:display-dependency-updates --fail-at-end -P jvm9-release -Djvm9.location=/home/travis/openjdk9
\ No newline at end of file
diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc
new file mode 100644
index 000000000..7e36bf009
--- /dev/null
+++ b/CHANGELOG.adoc
@@ -0,0 +1,393 @@
+:toc: macro
+:toclevels: 1
+
+= Change Log
+
+A high level summary of noteworthy changes in each version.
+
+NOTE:: Dependency version bumps are not listed here.
+
+// git log --pretty="* %s" 0.3.0.2..HEAD
+
+// only show TOC if this is the root document (not in the README)
+ifndef::github_name[]
+toc::[]
+endif::[]
+== 0.5.2.8
+
+=== Fixes
+
+* fix: Fix equality and hash code for ShardKey with array key (#638), resolves (#579)
+
+== 0.5.2.7
+
+=== Fixes
+
+* fix: Return cached pausedPartitionSet (#620), resolves (#618)
+* fix: Parallel consumer stops processing data sometimes (#623), fixes (#606)
+* fix: Add synchronization to ensure proper intializaiton and closing of PCMetrics singleton (#627), fixes (#617)
+* fix: Readme - metrics example correction (#614)
+* fix: Remove micrometer-atlas dependency (#628), fixes (#625)
+
+=== Improvements
+
+* Refactored metrics implementation to not use singleton - improves meter separation, allows correct metrics subsystem operation when multiple parallel consumer instances are running in same java process (#630), fixes (#617) improves on (#627)
+
+== 0.5.2.6
+=== Improvements
+
+* feature: Micrometer metrics (#594)
+* feature: Adds an option to pass an invalid offset metadata error policy (#537), improves (#326)
+* feature: Lazy intialization of workerThreadPool (#531)
+
+=== Fixes
+
+* fix: Don't drain mode shutdown kills inflight threads (#559)
+* fix: Drain mode shutdown doesn't pause consumption correctly (#552)
+* fix: RunLength offset decoding returns 0 base offset after no-progress commit - related to (#546)
+* fix: Transactional PConsumer stuck while rebalancing - related to (#541)
+
+=== Dependencies
+
+* PL-211: Update dependencies from dependabot, Add mvnw, use mvnw in jenkins (#583)
+* PL-211: Update dependencies from dependabot (#589)
+
+== 0.5.2.5
+
+=== Fixes
+
+* fixes: #195 NoSuchFieldException when using consumer inherited from KafkaConsumer (#469)
+* fix: After new performance fix PR#530 merges - corner case could cause out of order processing (#534)
+* fix: Cleanup WorkManager's count of in-progress work, when work is stale after partition revocation (#547)
+
+=== Improvements
+
+* perf: Adds a caching layer to work management to alleviate O(n) counting (#530)
+
+== 0.5.2.4
+
+=== Improvements
+
+* feature: Simple PCRetriableException to remove error spam from logs (#444)
+* minor: fixes #486: Missing generics in JStreamParallelStreamProcessor #491
+* minor: partially address #459: Moves isClosedOrFailed into top level ParallelConsumer interface (#491)
+* tests: Demonstrates how to use MockConsumer with PC for issue #176
+* other minor improvements
+
+=== Fixes
+
+* fixes #409: Adds support for compacted topics and commit offset resetting (#425)
+** Truncate the offset state when bootstrap polled offset higher or lower than committed
+** Prune missing records from the tracked incomplete offset state, when they're missing from polled batches
+* fix: Improvements to encoding ranges (int vs long) #439
+** Replace integer offset references with long - use Long everywhere we deal with offsets, and where we truncate down, do it exactly, detect and handle truncation issues.
+
+== 0.5.2.3
+
+=== Improvements
+
+* Transactional commit mode system improvements and docs (#355)
+** Clarifies transaction system with much better documentation.
+** Fixes a potential race condition which could cause offset leaks between transactions boundaries.
+** Introduces lock acquisition timeouts.
+** Fixes a potential issue with removing records from the retry queue incorrectly, by having an inconsistency between compareTo and equals in the retry TreeMap.
+* Adds a very simple Dependency Injection system modeled on Dagger (#398)
+* Various refactorings e.g. new ProducerWrap
+
+* Dependencies
+** build(deps): prod: zstd, reactor, dev: podam, progressbar, postgresql maven-plugins: versions, help (#420)
+** build(deps-dev): bump postgresql from 42.4.1 to 42.5.0
+** bump podam, progressbar, zstd, reactor
+** build(deps): bump versions-maven-plugin from 2.11.0 to 2.12.0
+** build(deps): bump maven-help-plugin from 3.2.0 to 3.3.0
+** build(deps-dev): bump Confluent Platform Kafka Broker to 7.2.2 (#421)
+** build(deps): Upgrade to AK 3.3.0 (#309)
+
+
+=== Fixes
+
+* fixes #419: NoSuchElementException during race condition in PartitionState (#422)
+* Fixes #412: ClassCastException with retryDelayProvider (#417)
+* fixes ShardManager retryQueue ordering and set issues due to poor Comparator implementation (#423)
+
+
+== v0.5.2.2
+
+=== Fixes
+
+- Fixes dependency scope for Mockito from compile to test (#376)
+
+== v0.5.2.1
+
+=== Fixes
+
+- Fixes regression issue with order of state truncation vs commit (#362)
+
+== v0.5.2.0
+
+=== Fixes and Improvements
+
+- fixes #184: Fix multi topic subscription with KEY order by adding topic to shard key (#315)
+- fixes #329: Committing around transaction markers causes encoder to crash (#328)
+- build: Upgrade Truth-Generator to 0.1.1 for user Subject discovery (#332)
+
+=== Build
+
+- build: Allow snapshots locally, fail in CI (#331)
+- build: OSS Index scan change to warn only and exclude Guava CVE-2020-8908 as it's WONT_FIX (#330)
+
+=== Dependencies
+
+- build(deps): bump reactor-core from 3.4.19 to 3.4.21 (#344)
+- build(deps): dependabot bump Mockito, Surefire, Reactor, AssertJ, Release (#342) (#342)
+- build(deps): dependabot bump TestContainers, Vert.x, Enforcer, Versions, JUnit, Postgress (#336)
+
+=== Linked issues
+
+- Message with null key lead to continuous failure when using KEY ordering #318
+- Subscribing to two or more topics with KEY ordering, results in messages of the same Key never being processed #184
+- Cannot have negative length BitSet error - committing transaction adjacent offsets #329
+
+== v0.5.1.0
+
+=== Features
+
+* #193: Pause / Resume PC (circuit breaker) without unsubscribing from topics
+
+=== Fixes and Improvements
+
+* #225: Build and runtime support for Java 16+ (#289)
+* #306: Change Truth-Generator dependency from compile to test
+* #298: Improve PollAndProduce performance by first producing all records, and then waiting for the produce results.Previously, this was done for each ProduceRecord individually.
+
+== v0.5.0.0
+
+=== Features
+
+* feature: Poll Context object for API (#223)
+** PollContext API - provides central access to result set with various convenience methods as well as metadata about records, such as failure count
+* major: Batching feature and Event system improvements
+** Batching - all API methods now support batching.
+See the Options class set batch size for more information.
+
+=== Fixes and Improvements
+
+* Event system - better CPU usage in control thread
+* Concurrency stability improvements
+* Update dependencies
+* #247: Adopt Truth-Generator (#249)
+** Adopt https://github.com/astubbs/truth-generator[Truth Generator] for automatic generation of https://truth.dev/[Google Truth] Subjects
+* Large rewrite of internal architecture for improved maintence and simplicity which fixed some corner case issues
+** refactor: Rename PartitionMonitor to PartitionStateManager (#269)
+** refactor: Queue unification (#219)
+** refactor: Partition state tracking instead of search (#218)
+** refactor: Processing Shard object
+* fix: Concurrency and State improvements (#190)
+
+=== Build
+
+* build: Lock TruthGenerator to 0.1 (#272)
+* build: Deploy SNAPSHOTS to maven central snaphots repo (#265)
+* build: Update Kafka to 3.1.0 (#229)
+* build: Crank up Enforcer rules and turn on ossindex audit
+* build: Fix logback dependency back to stable
+* build: Upgrade TestContainer and CP
+
+== v0.4.0.1
+
+=== Improvements
+
+- Add option to specify timeout for how long to wait offset commits in periodic-consumer-sync commit-mode
+- Add option to specify timeout for how long to wait for blocking Producer#send
+
+=== Docs
+
+- docs: Confluent Cloud configuration links
+- docs: Add Confluent's product page for PC to README
+- docs: Add head of line blocking to README
+
+== v0.4.0.0
+// https://github.com/confluentinc/parallel-consumer/releases/tag/0.4.0.0
+
+=== Features
+
+* https://projectreactor.io/[Project Reactor] non-blocking threading adapter module
+* Generic Vert.x Future support - i.e. FileSystem, db etc...
+
+=== Fixes and Improvements
+
+* Vert.x concurrency control via WebClient host limits fixed - see #maxCurrency
+* Vert.x API cleanup of invalid usage
+* Out of bounds for empty collections
+* Use ConcurrentSkipListMap instead of TreeMap to prevent concurrency issues under high pressure
+* log: Show record topic in slow-work warning message
+
+== v0.3.2.0
+
+=== Fixes and Improvements
+
+* Major: Upgrade to Apache Kafka 2.8 (still compatible with 2.6 and 2.7 though)
+* Adds support for managed executor service (Java EE Compatibility feature)
+* #65 support for custom retry delay providers
+
+== v0.3.1.0
+
+=== Fixes and Improvements
+
+* Major refactor to code base - primarily the two large God classes
+** Partition state now tracked separately
+** Code moved into packages
+* Busy spin in some cases fixed (lower CPU usage)
+* Reduce use of static data for test assertions - remaining identified for later removal
+* Various fixes for parallel testing stability
+
+== v0.3.0.3
+
+=== Fixes and Improvements
+
+==== Overview
+
+* Tests now run in parallel
+* License fixing / updating and code formatting
+* License format runs properly now when local, check on CI
+* Fix running on Windows and Linux
+* Fix JAVA_HOME issues
+
+==== Details:
+
+* tests: Enable the fail fast feature now that it's merged upstream
+* tests: Turn on parallel test runs
+* format: Format license, fix placement
+* format: Apply Idea formatting (fix license layout)
+* format: Update mycila license-plugin
+* test: Disable redundant vert.x test - too complicated to fix for little gain
+* test: Fix thread counting test by closing PC @After
+* test: Test bug due to static state overrides when run as a suite
+* format: Apply license format and run every All Idea build
+* format: Organise imports
+* fix: Apply license format when in dev laptops - CI only checks
+* fix: javadoc command for various OS and envs when JAVA_HOME missing
+* fix: By default, correctly run time JVM as jvm.location
+
+== v0.3.0.2
+
+=== Fixes and Improvements
+
+* ci: Add CODEOWNER
+* fix: #101 Validate GroupId is configured on managed consumer
+* Use 8B1DA6120C2BF624 GPG Key For Signing
+* ci: Bump jdk8 version path
+* fix: #97 Vert.x thread and connection pools setup incorrect
+* Disable Travis and Codecov
+* ci: Apache Kafka and JDK build matrix
+* fix: Set Serdes for MockProducer for AK 2.7 partition fix KAFKA-10503 to fix new NPE
+* Only log slow message warnings periodically, once per sweep
+* Upgrade Kafka container version to 6.0.2
+* Clean up stalled message warning logs
+* Reduce log-level if no results are returned from user-function (warn -> debug)
+* Enable java 8 Github
+* Fixes #87 - Upgrade UniJ version for UnsupportedClassVersion error
+* Bump TestContainers to stable release to specifically fix #3574
+* Clarify offset management capabilities
+
+== v0.3.0.1
+
+* fixes #62: Off by one error when restoring offsets when no offsets are encoded in metadata
+* fix: Actually skip work that is found as stale
+
+== v0.3.0.0
+
+=== Features
+
+* Queueing and pressure system now self tuning, performance over default old tuning values (`softMaxNumberMessagesBeyondBaseCommitOffset` and `maxMessagesToQueue`) has doubled.
+** These options have been removed from the system.
+* Offset payload encoding back pressure system
+** If the payload begins to take more than a certain threshold amount of the maximum available, no more messages will be brought in for processing, until the space need beings to reduce back below the threshold.
+This is to try to prevent the situation where the payload is too large to fit at all, and must be dropped entirely.
+** See Proper offset encoding back pressure system so that offset payloads can't ever be too large https://github.com/confluentinc/parallel-consumer/issues/47[#47]
+** Messages that have failed to process, will always be allowed to retry, in order to reduce this pressure.
+
+=== Improvements
+
+* Default ordering mode is now `KEY` ordering (was `UNORDERED`).
+** This is a better default as it's the safest mode yet high performing mode.
+It maintains the partition ordering characteristic that all keys are processed in log order, yet for most use cases will be close to as fast as `UNORDERED` when the key space is large enough.
+* https://github.com/confluentinc/parallel-consumer/issues/37[Support BitSet encoding lengths longer than Short.MAX_VALUE #37] - adds new serialisation formats that supports wider range of offsets - (32,767 vs 2,147,483,647) for both BitSet and run-length encoding.
+* Commit modes have been renamed to make it clearer that they are periodic, not per message.
+* Minor performance improvement, switching away from concurrent collections.
+
+=== Fixes
+
+* Maximum offset payload space increased to correctly not be inversely proportional to assigned partition quantity.
+* Run-length encoding now supports compacted topics, plus other bug fixes as well as fixes to Bitset encoding.
+
+== v0.2.0.3
+
+=== Fixes
+
+** https://github.com/confluentinc/parallel-consumer/issues/35[Bitset overflow check (#35)] - gracefully drop BitSet or Runlength encoding as an option if offset difference too large (short overflow)
+*** A new serialisation format will be added in next version - see https://github.com/confluentinc/parallel-consumer/issues/37[Support BitSet encoding lengths longer than Short.MAX_VALUE #37]
+** Gracefully drops encoding attempts if they can't be run
+** Fixes a bug in the offset drop if it can't fit in the offset metadata payload
+
+== v0.2.0.2
+
+=== Fixes
+
+** Turns back on the https://github.com/confluentinc/parallel-consumer/issues/35[Bitset overflow check (#35)]
+
+== v0.2.0.1 DO NOT USE - has critical bug
+
+=== Fixes
+
+** Incorrectly turns off an over-flow check in https://github.com/confluentinc/parallel-consumer/issues/35[offset serialisation system (#35)]
+
+== v0.2.0.0
+
+=== Features
+
+** Choice of commit modes: Consumer Asynchronous, Synchronous and Producer Transactions
+** Producer instance is now optional
+** Using a _transactional_ Producer is now optional
+** Use the Kafka Consumer to commit `offsets` Synchronously or Asynchronously
+
+=== Improvements
+
+** Memory performance - garbage collect empty shards when in KEY ordering mode
+** Select tests adapted to non transactional (multiple commit modes) as well
+** Adds supervision to broker poller
+** Fixes a performance issue with the async committer not being woken up
+** Make committer thread revoke partitions and commit
+** Have onPartitionsRevoked be responsible for committing on close, instead of an explicit call to commit by controller
+** Make sure Broker Poller now drains properly, committing any waiting work
+
+=== Fixes
+
+** Fixes bug in commit linger, remove genesis offset (0) from testing (avoid races), add ability to request commit
+** Fixes #25 https://github.com/confluentinc/parallel-consumer/issues/25:
+*** Sometimes a transaction error occurs - Cannot call send in state COMMITTING_TRANSACTION #25
+** ReentrantReadWrite lock protects non-thread safe transactional producer from incorrect multithreaded use
+** Wider lock to prevent transaction's containing produced messages that they shouldn't
+** Must start tx in MockProducer as well
+** Fixes example app tests - incorrectly testing wrong thing and MockProducer not configured to auto complete
+** Add missing revoke flow to MockConsumer wrapper
+** Add missing latch timeout check
+
+== v0.1
+
+=== Features:
+
+** Have massively parallel consumption processing without running hundreds or thousands of
+*** Kafka consumer clients
+*** topic partitions
++
+without operational burden or harming the clusters performance
+** Efficient individual message acknowledgement system (without local or third system state) to massively reduce message replay upon failure
+** Per `key` concurrent processing, per `partition` and unordered message processing
+** `Offsets` committed correctly, in order, of only processed messages, regardless of concurrency level or retries
+** Vert.x non-blocking library integration (HTTP currently)
+** Fair partition traversal
+** Zero~ dependencies (`Slf4j` and `Lombok`) for the core module
+** Java 8 compatibility
+** Throttle control and broker liveliness management
+** Clean draining shutdown cycle
diff --git a/Default.xml b/Default.xml
new file mode 100644
index 000000000..cd997255e
--- /dev/null
+++ b/Default.xml
@@ -0,0 +1,28 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 000000000..16a61caa7
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,66 @@
+#!/usr/bin/env groovy
+
+//common {
+// slackChannel = 'csid-build'
+// nodeLabel = 'docker-openjdk13'
+// runMergeCheck = false
+// mvnSkipDeploy = true
+}
+
+def RelaseTag = string(name: 'RELEASE_TAG', defaultValue: '',
+ description: 'Provide the tag of project that will be release to maven central,' +
+ 'only use the value when you want to release to maven central')
+
+def config = jobConfig {
+ owner = 'csid'
+// testResultSpecs = ['junit': 'test/results.xml']
+ properties = [parameters([RelaseTag])]
+ slackChannel = 'csid-build'
+ nodeLabel = 'docker-debian-jdk17'
+ runMergeCheck = true
+}
+
+def job = {
+ def maven_command = sh(script: """if test -f "${env.WORKSPACE}/mvnw"; then echo "${env.WORKSPACE}/mvnw"; else echo "mvn"; fi""", returnStdout: true).trim()
+ // If we have a RELEASE_TAG specified as a build parameter, test that the version in pom.xml matches the tag.
+ if (!params.RELEASE_TAG.trim().equals('')) {
+ sh "git checkout ${params.RELEASE_TAG}"
+ def project_version = sh(
+ script: """${maven_command} help:evaluate -Dexpression=project.version -q -DforceStdout | tail -1""",
+ returnStdout: true
+ ).trim()
+
+ if (!params.RELEASE_TAG.trim().equals(project_version)) {
+ echo 'ERROR: tag doesn\'t match project version, please correct and try again'
+ echo "Tag: ${params.RELEASE_TAG}"
+ echo "Project version: ${project_version}"
+ currentBuild.result = 'FAILURE'
+ return
+ }
+ }
+
+ stage('Build') {
+ archiveArtifacts artifacts: 'pom.xml'
+ withVaultEnv([["gpg/confluent-packaging-private-8B1DA6120C2BF624", "passphrase", "GPG_PASSPHRASE"]]) {
+ def mavenSettingsFile = "${env.WORKSPACE_TMP}/maven-global-settings.xml"
+ withMavenSettings("maven/jenkins_maven_global_settings", "settings", "MAVEN_GLOBAL_SETTINGS", mavenSettingsFile) {
+ withMaven(globalMavenSettingsFilePath: mavenSettingsFile) {
+ withDockerServer([uri: dockerHost()]) {
+ def isPrBuild = env.CHANGE_TARGET ? true : false
+ def buildPhase = isPrBuild ? "install" : "deploy"
+ if (params.RELEASE_TAG.trim().equals('')) {
+ sh "${maven_command} --batch-mode -Pjenkins -Pci -U dependency:analyze clean $buildPhase"
+ } else {
+ // it's a parameterized job, and we should deploy to maven central.
+ withGPGkey("gpg/confluent-packaging-private-8B1DA6120C2BF624") {
+ sh "${maven_command} --batch-mode clean deploy -P maven-central -Pjenkins -Pci -Dgpg.passphrase=$GPG_PASSPHRASE"
+ }
+ }
+ currentBuild.result = 'Success'
+ }
+ }
+ }
+ }
+ }
+}
+runJob config, job
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 000000000..7a4a3ea24
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
\ No newline at end of file
diff --git a/README.adoc b/README.adoc
new file mode 100644
index 000000000..81af418e0
--- /dev/null
+++ b/README.adoc
@@ -0,0 +1,1896 @@
+//
+// STOP!!! Make sure you're editing the TEMPLATE version of the README, in /src/docs/README_TEMPLATE.adoc
+//
+// Do NOT edit /README_TEMPLATE.adoc as your changes will be overwritten when the template is rendered again during
+// `process-sources`.
+//
+// Changes made to this template, must then be rendered to the base readme, by running `mvn process-sources`
+//
+// To render the README directly, run `mvn asciidoc-template::build`
+//
+
+
+// dynamic include base for editing in IDEA
+:project_root: ./
+// for editing the template to see the includes, this will correctly render includes
+ifeval::["{docname}" == "README_TEMPLATE"]
+
+TIP:: Editing template file
+
+:project_root: ../../
+
+endif::[]
+
+
+= Confluent Parallel Consumer
+:icons:
+:toc: macro
+:toclevels: 3
+:numbered: 1
+:sectlinks: true
+:sectanchors: true
+
+:github_name: parallel-consumer
+:base_url: https://github.com/confluentinc/{github_name}
+:issues_link: {base_url}/issues
+
+
+ifdef::env-github[]
+:tip-caption: :bulb:
+:note-caption: :information_source:
+:important-caption: :heavy_exclamation_mark:
+:caution-caption: :fire:
+:warning-caption: :warning:
+endif::[]
+
+image:https://maven-badges.herokuapp.com/maven-central/io.confluent.parallelconsumer/parallel-consumer-parent/badge.svg?style=flat[link=https://mvnrepository.com/artifact/io.confluent.parallelconsumer/parallel-consumer-parent,Latest Parallel Consumer on Maven Central]
+
+// Github actions disabled since codecov
+//image:https://github.com/confluentinc/parallel-consumer/actions/workflows/maven.yml/badge.svg[Java 8 Unit Test GitHub] +
+//^(^^full^ ^test^ ^suite^ ^currently^ ^running^ ^only^ ^on^ ^Confluent^ ^internal^ ^CI^ ^server^^)^
+
+// travis badges temporarily disabled as travis isn't running CI currently
+//image:https://travis-ci.com/astubbs/parallel-consumer.svg?branch=master["Build Status", link="https://travis-ci.com/astubbs/parallel-consumer"] image:https://codecov.io/gh/astubbs/parallel-consumer/branch/master/graph/badge.svg["Coverage",https://codecov.io/gh/astubbs/parallel-consumer]
+
+Parallel Apache Kafka client wrapper with client side queueing, a simpler consumer/producer API with *key concurrency* and *extendable non-blocking IO* processing.
+
+Confluent's https://www.confluent.io/confluent-accelerators/#parallel-consumer[product page for the project is here].
+
+TIP: If you like this project, please ⭐ Star it in GitHub to show your appreciation, help us gauge popularity of the project and allocate resources.
+
+NOTE: This is not a part of the Confluent commercial support offering, except through consulting engagements.
+See the <> section for more information.
+
+IMPORTANT: This project has been stable and reached its initial target feature set in Q1 2021.
+It is actively maintained by the CSID team at Confluent.
+
+[[intro]]
+This library lets you process messages in parallel via a single Kafka Consumer meaning you can increase consumer parallelism without increasing the number of partitions in the topic you intend to process.
+For many use cases this improves both throughput and latency by reducing load on your brokers.
+It also opens up new use cases like extreme parallelism, external data enrichment, and queuing.
+
+.Consume many messages _concurrently_ with a *single* consumer instance:
+[source,java,indent=0]
+----
+ parallelConsumer.poll(record ->
+ log.info("Concurrently processing a record: {}", record)
+ );
+----
+
+An overview article to the library can also be found on Confluent's https://www.confluent.io/blog/[blog]: https://www.confluent.io/blog/introducing-confluent-parallel-message-processing-client/[Introducing the Confluent Parallel Consumer].
+
+[#demo]
+== Demo
+
+.Relative speed demonstration
+--
+.Click on the animated SVG image to open the https://asciinema.org/a/404299[Asciinema.org player].
+image::https://gist.githubusercontent.com/astubbs/26cccaf8b624a53ae26a52dbc00148b1/raw/cbf558b38b0aa624bd7637406579d2a8f00f51db/demo.svg[link="https://asciinema.org/a/404299"]
+--
+
+:talk_link: https://www.confluent.io/en-gb/events/kafka-summit-europe-2021/introducing-confluent-labs-parallel-consumer-client/
+:talk_preview_image: https://play.vidyard.com/5MLb1Xh7joEQ7phxPxiyPK.jpg
+
+[#talk]
+== Video Overview
+
+.Kafka Summit Europe 2021 Presentation
+--
+.A video presentation overview can be found {talk_link}[from the Kafka Summit Europe 2021] page for the presentatoin, along with slides.
+[link = {talk_link}]
+image::{talk_preview_image}[Talk]
+--
+
+'''
+
+toc::[]
+
+== Motivation
+
+=== Why would I need this?
+
+The unit of parallelism in Kafka’s consumers is the partition but sometimes you want to break away from this approach and manage parallelism yourself using threads rather than new instances of a Consumer.
+Notable use cases include:
+
+* Where partition counts are difficult to change and you need more parallelism than the current configuration allows.
+
+* You wish to avoid over provisioning partitions in topics due to unknown future requirements.
+
+* You wish to reduce the broker-side resource utilization associated with highly-parallel consumer groups.
+
+* You need queue-like semantics that use message level acknowledgment, for example to process a work queue with short- and long-running tasks.
+
+When reading the below, keep in mind that the unit of concurrency and thus performance, is restricted by the number of partitions (degree of sharding / concurrency).
+Currently, you can't adjust the number of partitions in your Kafka topics without jumping through a lot of hoops, or breaking your key ordering.
+
+==== Before
+
+.The slow consumer situation with the raw Apache Kafka Consumer client
+image::https://lucid.app/publicSegments/view/98ad200f-97b2-479b-930c-2805491b2ce7/image.png[align="center"]
+
+==== After
+
+.Example usage of the Parallel Consumer
+image::https://lucid.app/publicSegments/view/2cb3b7e2-bfdf-4e78-8247-22ec394de965/image.png[align="center"]
+
+=== Background
+
+The core Kafka consumer client gives you a batch of messages to process one at a time.
+Processing these in parallel on thread pools is difficult, particularly when considering offset management and strong ordering guarantees.
+You also need to manage your consume loop, and commit transactions properly if using Exactly Once semantics.
+
+This wrapper library for the Apache Kafka Java client handles all this for you, you just supply your processing function.
+
+Another common situation where concurrent processing of messages is advantageous, is what is referred to as "competing consumers".
+A pattern that is often addressed in traditional messaging systems using a shared queue.
+Kafka doesn't provide native queue support and this can result in a slow processing message blocking the messages behind it in the same partition.
+If <> isn't a concern this can be an unwelcome bottleneck for users.
+The Parallel Consumer provides a solution to this problem.
+
+In addition, the <> to this library supplies non-blocking interfaces, allowing higher still levels of concurrency with a further simplified interface.
+Also included now is a <> https://projectreactor.io[Project Reactor.io].
+
+=== FAQ
+
+[qanda]
+Why not just run more consumers?::
+The typical way to address performance issues in a Kafka system, is to increase the number of consumers reading from a topic.
+This is effective in many situations, but falls short in a lot too.
+
+* Primarily: You cannot use more consumers than you have partitions available to read from.
+For example, if you have a topic with five partitions, you cannot use a group with more than five consumers to read from it.
+* Running more extra consumers has resource implications - each consumer takes up resources on both the client and broker side.
+Each consumer adds a lot of overhead in terms of memory, CPU, and network bandwidth.
+* Large consumer groups (especially many large groups) can cause a lot of strain on the consumer group coordination system, such as rebalance storms.
+* Even with several partitions, you cannot achieve the performance levels obtainable by *per-key* ordered or unordered concurrent processing.
+* A single slow or failing message will also still block all messages behind the problematic message, ie. the entire partition.
+The process may recover, but the latency of all the messages behind the problematic one will be negatively impacted severely.
+
+Why not run more consumers __within__ your application instance?::
+* This is in some respects a slightly easier way of running more consumer instances, and in others a more complicated way.
+However, you are still restricted by all the per consumer restrictions as described above.
+
+Why not use the Vert.x library yourself in your processing loop?::
+* Vert.x us used in this library to provide a non-blocking IO system in the message processing step.
+Using Vert.x without using this library with *ordered* processing requires dealing with the quite complicated, and not straight forward, aspect of handling offset commits with Vert.x asynchronous processing system.
++
+*Unordered* processing with Vert.x is somewhat easier, however offset management is still quite complicated, and the Parallel Consumer also provides optimizations for message-level acknowledgment in this case.
+This library handles offset commits for both ordered and unordered processing cases.
+
+=== Scenarios
+
+Below are some real world use cases which illustrate concrete situations where the described advantages massively improve performance.
+
+* Slow consumer systems in transactional systems (online vs offline or reporting systems)
+** Notification system:
++
+*** Notification processing system which sends push notifications to a user to acknowledge a two-factor authentication request on their mobile and authorising a login to a website, requires optimal end-to-end latency for a good user experience.
+*** A specific message in this queue uncharacteristically takes a long time to process because the third party system is sometimes unpredictably slow to respond and so holds up the processing for *ALL* other notifications for other users that are in the same partition behind this message.
+*** Using key order concurrent processing will allow notifications to proceed while this message either slowly succeeds or times out and retires.
+** Slow GPS tracking system (slow HTTP service interfaces that can scale horizontally)
+*** GPS tracking messages from 100,000 different field devices pour through at a high rate into an input topic.
+*** For each message, the GPS location coordinates is checked to be within allowed ranges using a legacy HTTP services, dictated by business rules behind the service.
+*** The service takes 50ms to process each message, however can be scaled out horizontally without restriction.
+*** The input topic only has 10 partitions and for various reasons (see above) cannot be changed.
+*** With the vanilla consumer, messages on each partition must be consumed one after the other in serial order.
+*** The maximum rate of message processing is then:
++
+`1 second / 50 ms * 10 partitions = 200 messages per second.`
+*** By using this library, the 10 partitions can all be processed in key order.
++
+`1 second / 50ms × 100,000 keys = 2,000,000 messages per second`
++
+While the HTTP system probably cannot handle 2,000,000 messages per second, more importantly, your system is no longer the bottleneck.
+
+** Slow CPU bound model processing for fraud prediction
+*** Consider a system where message data is passed through a fraud prediction model which takes CPU cycles, instead of an external system being slow.
+*** We can scale easily the number of CPUs on our virtual machine where the processing is being run, but we choose not to scale the partitions or consumers (see above).
+*** By deploying onto machines with far more CPUs available, we can run our prediction model massively parallel, increasing our throughput and reducing our end-to-end response times.
+* Spikey load with latency sensitive non-functional requirements
+** An upstream system regularly floods our input topic daily at close of business with settlement totals data from retail outlets.
+*** Situations like this are common where systems are designed to comfortably handle average day time load, but are not provisioned to handle sudden increases in traffic as they don't happen often enough to justify the increased spending on processing capacity that would otherwise remain idle.
+*** Without adjusting the available partitions or running consumers, we can reduce our maximum end-to-end latency and increase throughout to get our global days outlet reports to division managers so action can be taken, before close of business.
+** Natural consumer behaviour
+*** Consider scenarios where bursts of data flooding input topics are generated by sudden user behaviour such as sales or television events ("Oprah" moments).
+*** For example, an evening, prime-time game show on TV where users send in quiz answers on their devices.
+The end-to-end latency of the responses to these answers needs to be as low as technically possible, even if the processing step is quick.
+*** Instead of a vanilla client where each user response waits in a virtual queue with others to be processed, this library allows every single response to be processed in parallel.
+* Legacy partition structure
+** Any existing setups where we need higher performance either in throughput or latency where there are not enough partitions for needed concurrency level, the tool can be applied.
+* Partition overloaded brokers
+** Clusters with under-provisioned hardware and with too many partitions already - where we cannot expand partitions even if we were able to.
+** Similar to the above, but from the operations perspective, our system is already over partitioned, perhaps in order to support existing parallel workloads which aren't using the tool (and so need large numbers of partitions).
+** We encourage our development teams to migrate to the tool, and then being a process of actually __lowering__ the number of partitions in our partitions in order to reduce operational complexity, improve reliability and perhaps save on infrastructure costs.
+* Server side resources are controlled by a different team we can't influence
+** The cluster our team is working with is not in our control, we cannot change the partition setup, or perhaps even the consumer layout.
+** We can use the tool ourselves to improve our system performance without touching the cluster / topic setup.
+* Kafka Streams app that had a slow stage
+** We use Kafka Streams for our message processing, but one of it's steps have characteristics of the above and we need better performance.
+We can break out as described below into the tool for processing that step, then return to the Kafka Streams context.
+* Provisioning extra machines (either virtual machines or real machines) to run multiple clients has a cost, using this library instead avoids the need for extra instances to be deployed in any respect.
+
+== Features List
+
+* Have massively parallel consumption processing without running hundreds or thousands of:
+** Kafka consumer clients,
+** topic partitions,
++
+without operational burden or harming the cluster's performance
+* Client side queueing system on top of Apache Kafka consumer
+** Efficient individual message acknowledgement system (without local or third party external system state storage) to massively reduce (and usually completely eliminate) message replay upon failure - see <> section for more details
+* Solution for the https://en.wikipedia.org/wiki/Head-of-line_blocking["head of line"] blocking problem where continued failure of a single message, prevents progress for messages behind it in the queue
+* Per `key` concurrent processing, per partition and unordered message processing
+* Offsets committed correctly, in order, of only processed messages, regardless of concurrency level or retries
+* Batch support in all versions of the API to process batches of messages in parallel instead of single messages.
+** Particularly useful for when your processing function can work with more than a single record at a time - e.g. sending records to an API which has a batch version like Elasticsearch
+* Vert.x and Reactor.io non-blocking library integration
+** Non-blocking I/O work management
+** Vert.x's WebClient and general Vert.x Future support
+** Reactor.io Publisher (Mono/Flux) and Java's CompletableFuture (through `Mono#fromFuture`)
+* Exactly Once bulk transaction system
+** When using the transactional mode, record processing that happens in parallel and produce records back to kafka get all grouped into a large batch transaction, and the offsets and records are submitted through the transactional producer, giving you Exactly once Semantics for parallel processing.
+** For further information, see the <> section.
+* Fair partition traversal
+* Zero~ dependencies (`Slf4j` and `Lombok`) for the core module
+* Java 8 compatibility
+* Throttle control and broker liveliness management
+* Clean draining shutdown cycle
+* Manual global pause / resume of all partitions, without unsubscribing from topics (useful for implementing a simplistic https://en.wikipedia.org/wiki/Circuit_breaker_design_pattern[circuit breaker])
+** Circuit breaker patterns for individual paritions or keys can be done through throwing failure exceptions in the processing function (see https://github.com/confluentinc/parallel-consumer/pull/291[PR #291 Explicit terminal and retriable exceptions] for further refinement)
+** Note: Pausing of a partition is also automatic, whenever back pressure has built up on a given partition
+
+//image:https://codecov.io/gh/astubbs/parallel-consumer/branch/master/graph/badge.svg["Coverage",https://codecov.io/gh/astubbs/parallel-consumer]
+//image:https://travis-ci.com/astubbs/parallel-consumer.svg?branch=master["Build Status", link="https://travis-ci.com/astubbs/parallel-consumer"]
+
+And more <>!
+
+== Performance
+
+In the best case, you don't care about ordering at all.In which case, the degree of concurrency achievable is simply set by max thread and concurrency settings, or with the Vert.x extension, the Vert.x Vertical being used - e.g. non-blocking HTTP calls.
+
+For example, instead of having to run 1,000 consumers to process 1,000 messages at the same time, we can process all 1,000 concurrently on a single consumer instance.
+
+More typically though you probably still want the per key ordering grantees that Kafka provides.
+For this there is the per key ordering setting.
+This will limit the library from processing any message at the same time or out of order, if they have the same key.
+
+Massively reduce message processing latency regardless of partition count for spikey workloads where there is good key distribution.
+Eg 100,000 “users” all trigger an action at once.
+As long as the processing layer can handle the load horizontally (e.g auto scaling web service), per message latency will be massively decreased, potentially down to the time for processing a single message, if the integration point can handle the concurrency.
+
+For example, if you have a key set of 10,000 unique keys, and you need to call an http endpoint to process each one, you can use the per key order setting, and in the best case the system will process 10,000 at the same time using the non-blocking Vert.x HTTP client library.
+The user just has to provide a function to extract from the message the HTTP call parameters and construct the HTTP request object.
+
+=== Illustrative Performance Example
+
+.(see link:./parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VolumeTests.java[VolumeTests.java])
+These performance comparison results below, even though are based on real performance measurement results, are for illustrative purposes.
+To see how the performance of the tool is related to instance counts, partition counts, key distribution and how it would relate to the vanilla client.
+Actual results will vary wildly depending upon the setup being deployed into.
+
+For example, if you have hundreds of thousands of keys in your topic, randomly distributed, even with hundreds of partitions, with only a handful of this wrapper deployed, you will probably see many orders of magnitude performance improvements - massively out performing dozens of vanilla Kafka consumer clients.
+
+.Time taken to process a large number of messages with a Single Parallel Consumer vs a single Kafka Consumer, for different key space sizes. As the number of unique keys in the data set increases, the key ordered Parallel Consumer performance starts to approach that of the unordered Parallel Consumer. The raw Kafka consumer performance remains unaffected by the key distribution.
+image::https://docs.google.com/spreadsheets/d/e/2PACX-1vQffkAFG-_BzH-LKfGCVnytdzAHiCNIrixM6X2vF8cqw2YVz6KyW3LBXTB-lVazMAJxW0UDuFILKvtK/pubchart?oid=1691474082&format=image[align="center"]
+
+.Consumer group size effect on total processing time vs a single Parallel Consumer. As instances are added to the consumer group, it's performance starts to approach that of the single instance Parallel Consumer. Key ordering is faster than partition ordering, with unordered being the fastest.
+image::https://docs.google.com/spreadsheets/d/e/2PACX-1vQffkAFG-_BzH-LKfGCVnytdzAHiCNIrixM6X2vF8cqw2YVz6KyW3LBXTB-lVazMAJxW0UDuFILKvtK/pubchart?oid=938493158&format=image[align="center"]
+
+.Consumer group size effect on message latency vs a single Parallel Consumer. As instances are added to the consumer group, it's performance starts to approach that of the single instance Parallel Consumer.
+image::https://docs.google.com/spreadsheets/d/e/2PACX-1vQffkAFG-_BzH-LKfGCVnytdzAHiCNIrixM6X2vF8cqw2YVz6KyW3LBXTB-lVazMAJxW0UDuFILKvtK/pubchart?oid=1161363385&format=image[align="center"]
+
+As an illustrative example of relative performance, given:
+
+* A random processing time between 0 and 5ms
+* 10,000 messages to process
+* A single partition (simplifies comparison - a topic with 5 partitions is the same as 1 partition with a keyspace of 5)
+* Default `ParallelConsumerOptions`
+** maxConcurrency = 100
+** numberOfThreads = 16
+
+.Comparative performance of order modes and key spaces
+[cols="1,1,1,3",options="header"]
+|===
+|Ordering
+|Number of keys
+|Duration
+|Note
+
+|Partition
+|20 (not relevant)
+|22.221s
+|This is the same as a single partition with a single normal serial consumer, as we can see: 2.5ms avg processing time * 10,000 msg / 1000ms = ~25s.
+
+|Key
+|1
+|26.743s
+|Same as above
+
+|Key
+|2
+|13.576s
+|
+
+|Key
+|5
+|5.916s
+|
+
+|Key
+|10
+|3.310s
+|
+
+|Key
+|20
+|2.242s
+|
+
+|Key
+|50
+|2.204s
+|
+
+|Key
+|100
+|2.178s
+|
+
+|Key
+|1,000
+|2.056s
+|
+
+|Key
+|10,000
+|2.128s
+|As key space is t he same as the number of messages, this is similar (but restricted by max concurrency settings) as having a *single consumer* instance and *partition* _per key_. 10,000 msgs * avg processing time 2.5ms = ~2.5s.
+
+|Unordered
+|20 (not relevant)
+|2.829s
+|As there is no order restriction, this is similar (but restricted by max concurrency settings) as having a *single consumer* instance and *partition* _per key_. 10,000 msgs * avg processing time 2.5ms = ~2.5s.
+|===
+
+== Support and Issues
+
+If you encounter any issues, or have any suggestions or future requests, please create issues in the {issues_link}[github issue tracker].
+Issues will be dealt with on a good faith, best efforts basis, by the small team maintaining this library.
+
+We also encourage participation, so if you have any feature ideas etc, please get in touch, and we will help you work on submitting a PR!
+
+NOTE: We are very interested to hear about your experiences!
+And please vote on your favourite issues!
+
+If you have questions, head over to the https://launchpass.com/confluentcommunity[Confluent Slack community], or raise an https://github.com/confluentinc/parallel-consumer/issues[issue] on GitHub.
+
+== License
+
+This library is copyright Confluent Inc, and licensed under the Apache License Version 2.0.
+
+== Usage
+
+=== Maven
+
+This project is available in maven central, https://repo1.maven.org/maven2/io/confluent/parallelconsumer/[repo1], along with SNAPSHOT builds (starting with 0.5-SNAPSHOT) in https://oss.sonatype.org/content/repositories/snapshots/io/confluent/parallelconsumer/[repo1's SNAPSHOTS repo].
+
+Latest version can be seen https://search.maven.org/artifact/io.confluent.parallelconsumer/parallel-consumer-core[here].
+
+Where `${project.version}` is the version to be used:
+
+* group ID: `io.confluent.parallelconsumer`
+* artifact ID: `parallel-consumer-core`
+* version: image:https://maven-badges.herokuapp.com/maven-central/io.confluent.parallelconsumer/parallel-consumer-parent/badge.svg?style=flat[link=https://mvnrepository.com/artifact/io.confluent.parallelconsumer/parallel-consumer-parent,Latest Parallel Consumer on Maven Central]
+
+.Core Module Dependency
+[source,xml,indent=0]
+
+ io.confluent.parallelconsumer
+ parallel-consumer-core
+ ${project.version}
+
+
+.Reactor Module Dependency
+[source,xml,indent=0]
+
+ io.confluent.parallelconsumer
+ parallel-consumer-reactor
+ ${project.version}
+
+
+.Vert.x Module Dependency
+[source,xml,indent=0]
+
+ io.confluent.parallelconsumer
+ parallel-consumer-vertx
+ ${project.version}
+
+
+[[common_preparation]]
+=== Common Preparation
+
+.Setup the client
+[source,java,indent=0]
+----
+ Consumer kafkaConsumer = getKafkaConsumer(); // <1>
+ Producer kafkaProducer = getKafkaProducer();
+
+ var options = ParallelConsumerOptions.builder()
+ .ordering(KEY) // <2>
+ .maxConcurrency(1000) // <3>
+ .consumer(kafkaConsumer)
+ .producer(kafkaProducer)
+ .build();
+
+ ParallelStreamProcessor eosStreamProcessor =
+ ParallelStreamProcessor.createEosStreamProcessor(options);
+
+ eosStreamProcessor.subscribe(of(inputTopic)); // <4>
+
+ return eosStreamProcessor;
+----
+
+<1> Setup your clients as per normal.
+A Producer is only required if using the `produce` flows.
+<2> Choose your ordering type, `KEY` in this case.
+This ensures maximum concurrency, while ensuring messages are processed and committed in `KEY` order, making sure no offset is committed unless all offsets before it in it's partition, are completed also.
+<3> The maximum number of concurrent processing operations to be performing at any given time.
+Also, because the library coordinates offsets, `enable.auto.commit` must be disabled in your consumer.
+<5> Subscribe to your topics
+
+NOTE: Because the library coordinates offsets, `enable.auto.commit` must be disabled.
+
+After this setup, one then has the choice of interfaces:
+
+* `ParallelStreamProcessor`
+* `VertxParallelStreamProcessor`
+* `JStreamParallelStreamProcessor`
+* `JStreamVertxParallelStreamProcessor`
+
+There is another interface: `ParallelConsumer` which is integrated, however there is currently no immediate implementation.
+See {issues_link}/12[issue #12], and the `ParallelConsumer` JavaDoc:
+
+[source,java]
+----
+/**
+ * Asynchronous / concurrent message consumer for Kafka.
+ *
+ * Currently, there is no direct implementation, only the {@link ParallelStreamProcessor} version (see
+ * {@link AbstractParallelEoSStreamProcessor}), but there may be in the future.
+ *
+ * @param key consume / produce key type
+ * @param value consume / produce value type
+ * @see AbstractParallelEoSStreamProcessor
+ */
+----
+
+=== Core
+
+==== Simple Message Process
+
+This is the only thing you need to do, in order to get massively concurrent processing in your code.
+
+.Usage - print message content out to the console in parallel
+[source,java,indent=0]
+ parallelConsumer.poll(record ->
+ log.info("Concurrently processing a record: {}", record)
+ );
+
+See the link:{project_root}/parallel-consumer-examples/parallel-consumer-example-core/src/main/java/io/confluent/parallelconsumer/examples/core/CoreApp.java[core example] project, and it's test.
+
+==== Process and Produce a Response Message
+
+This interface allows you to process your message, then publish back to the broker zero, one or more result messages.
+You can also optionally provide a callback function to be run after the message(s) is(are) successfully published to the broker.
+
+.Usage - print message content out to the console in parallel
+[source,java,indent=0]
+ parallelConsumer.pollAndProduce(context -> {
+ var consumerRecord = context.getSingleRecord().getConsumerRecord();
+ var result = processBrokerRecord(consumerRecord);
+ return new ProducerRecord<>(outputTopic, consumerRecord.key(), result.payload);
+ }, consumeProduceResult -> {
+ log.debug("Message {} saved to broker at offset {}",
+ consumeProduceResult.getOut(),
+ consumeProduceResult.getMeta().offset());
+ }
+ );
+
+==== Callbacks vs Streams
+
+You have the option to either use callbacks to be notified of events, or use the `Streaming` versions of the API, which use the `java.util.stream.Stream` system:
+
+* `JStreamParallelStreamProcessor`
+* `JStreamVertxParallelStreamProcessor`
+
+In future versions, we plan to look at supporting other streaming systems like https://github.com/ReactiveX/RxJava[RxJava] via modules.
+
+[[batching]]
+=== Batching
+
+The library also supports sending a batch or records as input to the users processing function in parallel.
+Using this, you can process several records in your function at once.
+
+To use it, set a `batch size` in the options class.
+
+There are then various access methods for the batch of records - see the `PollContext` object for more information.
+
+IMPORTANT: If an exception is thrown while processing the batch, all messages in the batch will be returned to the queue, to be retried with the standard retry system.
+There is no guarantee that the messages will be retried again in the same batch.
+
+==== Usage
+
+[source,java,indent=0]
+----
+ ParallelStreamProcessor.createEosStreamProcessor(ParallelConsumerOptions.builder()
+ .consumer(getKafkaConsumer())
+ .producer(getKafkaProducer())
+ .maxConcurrency(100)
+ .batchSize(5) // <1>
+ .build());
+ parallelConsumer.poll(context -> {
+ // convert the batch into the payload for our processing
+ List payload = context.stream()
+ .map(this::preparePayload)
+ .collect(Collectors.toList());
+ // process the entire batch payload at once
+ processBatchPayload(payload);
+ });
+----
+
+<1> Choose your batch size.
+
+==== Restrictions
+
+- If using a batch version of the API, you must choose a batch size in the options class.
+- If a batch size is chosen, the "normal" APIs cannot be used, and an error will be thrown.
+
+[[http-with-vertx]]
+=== HTTP with the Vert.x Module
+
+.Call an HTTP endpoint for each message usage
+[source,java,indent=0]
+----
+ var resultStream = parallelConsumer.vertxHttpReqInfoStream(context -> {
+ var consumerRecord = context.getSingleConsumerRecord();
+ log.info("Concurrently constructing and returning RequestInfo from record: {}", consumerRecord);
+ Map params = UniMaps.of("recordKey", consumerRecord.key(), "payload", consumerRecord.value());
+ return new RequestInfo("localhost", port, "/api", params); // <1>
+ });
+----
+
+<1> Simply return an object representing the request, the Vert.x HTTP engine will handle the rest, using it's non-blocking engine
+
+See the link:{project_root}/parallel-consumer-examples/parallel-consumer-example-vertx/src/main/java/io/confluent/parallelconsumer/examples/vertx/VertxApp.java[Vert.x example] project, and it's test.
+
+[[project-reactor]]
+=== Project Reactor
+
+As per the Vert.x support, there is also a Reactor module.
+This means you can use Reactor's non-blocking threading model to process your messages, allowing for orders of magnitudes higher concurrent processing than the core module's thread per worker module.
+
+See the link:{project_root}/parallel-consumer-examples/parallel-consumer-example-reactor/src/main/java/io/confluent/parallelconsumer/examples/reactor/ReactorApp.java[Reactor example] project, and it's test.
+
+.Call any Reactor API for each message usage. This example uses a simple `Mono.just` to return a value, but you can use any Reactor API here.
+[source,java,indent=0]
+----
+ parallelConsumer.react(context -> {
+ var consumerRecord = context.getSingleRecord().getConsumerRecord();
+ log.info("Concurrently constructing and returning RequestInfo from record: {}", consumerRecord);
+ Map params = UniMaps.of("recordKey", consumerRecord.key(), "payload", consumerRecord.value());
+ return Mono.just("something todo"); // <1>
+ });
+----
+
+[[spring]]
+[[streams-usage-code]]
+=== Kafka Streams Concurrent Processing
+
+Use your Streams app to process your data first, then send anything needed to be processed concurrently to an output topic, to be consumed by the parallel consumer.
+
+.Example usage with Kafka Streams
+image::https://lucid.app/publicSegments/view/43f2740c-2a7f-4b7f-909e-434a5bbe3fbf/image.png[Kafka Streams Usage,align="center"]
+
+.Preprocess in Kafka Streams, then process concurrently
+[source,java,indent=0]
+----
+ void run() {
+ preprocess(); // <1>
+ concurrentProcess(); // <2>
+ }
+
+ void preprocess() {
+ StreamsBuilder builder = new StreamsBuilder();
+ builder.stream(inputTopic)
+ .mapValues((key, value) -> {
+ log.info("Streams preprocessing key: {} value: {}", key, value);
+ return String.valueOf(value.length());
+ })
+ .to(outputTopicName);
+
+ startStreams(builder.build());
+ }
+
+ void startStreams(Topology topology) {
+ streams = new KafkaStreams(topology, getStreamsProperties());
+ streams.start();
+ }
+
+ void concurrentProcess() {
+ setupParallelConsumer();
+
+ parallelConsumer.poll(record -> {
+ log.info("Concurrently processing a record: {}", record);
+ messageCount.getAndIncrement();
+ });
+ }
+----
+
+<1> Setup your Kafka Streams stage as per normal, performing any type of preprocessing in Kafka Streams
+<2> For the slow consumer part of your Topology, drop down into the parallel consumer, and use massive concurrency
+
+See the link:{project_root}/parallel-consumer-examples/parallel-consumer-example-streams/src/main/java/io/confluent/parallelconsumer/examples/streams/StreamsApp.java[Kafka Streams example] project, and it's test.
+
+[[confluent-cloud]]
+=== Confluent Cloud
+
+. Provision your fully managed Kafka cluster in Confluent Cloud
+.. Sign up for https://www.confluent.io/confluent-cloud/tryfree/[Confluent Cloud], a fully-managed Apache Kafka service.
+.. After you log in to Confluent Cloud, click on `Add cloud environment` and name the environment `learn-kafka`.
+Using a new environment keeps your learning resources separate from your other Confluent Cloud resources.
+.. Click on https://confluent.cloud/learn[LEARN] and follow the instructions to launch a Kafka cluster and to enable Schema Registry.
+. Access the client configuration settings
+.. From the Confluent Cloud Console, navigate to your Kafka cluster.
+From the `Clients` view, get the connection information customized to your cluster (select `Java`).
+.. Create new credentials for your Kafka cluster, and then Confluent Cloud will show a configuration block with your new credentials automatically populated (make sure `show API keys` is checked).
+.. Use these settings presented to https://docs.confluent.io/clients-kafka-java/current/overview.html[configure your clients].
+. Use these clients for steps outlined in the <> section.
+
+[[upgrading]]
+== Upgrading
+
+=== From 0.4 to 0.5
+
+This version has a breaking change in the API - instead of passing in `ConsumerRecord` instances, it passes in a `PollContext` object which has extra information and utility methods.
+See the `PollContext` class for more information.
+
+[[ordering-guarantees]]
+== Ordering Guarantees
+
+The user has the option to either choose ordered, or unordered message processing.
+
+Either in `ordered` or `unordered` processing, the system will only commit offsets for messages which have been successfully processed.
+
+CAUTION: `Unordered` processing could cause problems for third party integration where ordering by key is required.
+
+CAUTION: Beware of third party systems which are not idempotent, or are key order sensitive.
+
+IMPORTANT: The below diagrams represent a single iteration of the system and a very small number of input partitions and messages.
+
+=== Vanilla Kafka Consumer Operation
+
+Given this input topic with three partitions and a series of messages:
+
+.Input topic
+image::https://lucid.app/publicSegments/view/37d13382-3067-4c93-b521-7e43f2295fff/image.png[align="center"]
+
+The normal Kafka client operations in the following manner.
+Note that typically offset commits are not performed after processing a single message, but is illustrated in this manner for comparison to the single pass concurrent methods below.
+Usually many messages are committed in a single go, which is much more efficient, but for our illustrative purposes is not really relevant, as we are demonstration sequential vs concurrent _processing_ messages.
+
+.Normal execution of the raw Kafka client
+image::https://lucid.app/publicSegments/view/0365890d-e8ff-4a06-b24a-8741175dacc3/image.png[align="center"]
+
+=== Unordered
+
+Unordered processing is where there is no restriction on the order of multiple messages processed per partition, allowing for highest level of concurrency.
+
+This is the fastest option.
+
+.Unordered concurrent processing of message
+image::https://lucid.app/publicSegments/view/aab5d743-de05-46d0-8c1e-0646d7d2946f/image.png[align="center"]
+
+=== Ordered by Partition
+
+At most only one message from any given input partition will be in flight at any given time.
+This means that concurrent processing is restricted to the number of input partitions.
+
+The advantage of ordered processing mode, is that for an assignment of 1000 partitions to a single consumer, you do not need to run 1000 consumer instances or threads, to process the partitions in parallel.
+
+Note that for a given partition, a slow processing message _will_ prevent messages behind it from being processed.
+However, messages in other partitions assigned to the consumer _will_ continue processing.
+
+This option is most like normal operation, except if the consumer is assigned more than one partition, it is free to process all partitions in parallel.
+
+.Partition ordered concurrent processing of messages
+image::https://lucid.app/publicSegments/view/30ad8632-e8fe-4e05-8afd-a2b6b3bab309/image.png[align="center"]
+
+=== Ordered by Key
+
+Most similar to ordered by partition, this mode ensures process ordering by *key* (per partition).
+
+The advantage of this mode, is that a given input topic may not have many partitions, it may have a ~large number of unique keys.
+Each of these key -> message sets can actually be processed concurrently, bringing concurrent processing to a per key level, without having to increase the number of input partitions, whilst keeping strong ordering by key.
+
+As usual, the offset tracking will be correct, regardless of the ordering of unique keys on the partition or adjacency to the committed offset, such that after failure or rebalance, the system will not replay messages already marked as successful.
+
+This option provides the performance of maximum concurrency, while maintaining message processing order per key, which is sufficient for many applications.
+
+.Key ordering concurrent processing of messages
+image::https://lucid.app/publicSegments/view/f7a05e99-24e6-4ea3-b3d0-978e306aa568/image.png[align="center"]
+
+=== Retries and Ordering
+
+Even during retries, offsets will always be committed only after successful processing, and in order.
+
+== Retries
+
+If processing of a record fails, the record will be placed back into it's queue and retried with a configurable delay (see the `ParallelConsumerOptions` class).
+Ordering guarantees will always be adhered to, regardless of failure.
+
+A failure is denoted by *any* exception being thrown from the user's processing function.
+The system catches these exceptions, logs them and replaces the record in the queue for processing later.
+All types of Exceptions thrown are considered retriable.
+To not retry a record, do not throw an exception from your processing function.
+
+TIP:: To avoid the system logging an error, throw an exception which extends PCRetriableException.
+
+TIP:: If there was an error processing a record, and you'd like to skip it - do not throw an exception, and the system will mark the record as succeeded.
+
+If for some reason you want to proactively fail a record, without relying on some other system throwing an exception which you don't catch - simply throw an exception of your own design, which the system will treat the same way.
+
+To configure the retry delay, see `ParallelConsumerOptions#defaultRetryDelay`.
+
+At the moment there is no terminal error support, so messages will continue to be retried forever as long as an exception continues to be thrown from the user function (see <>).
+But still this will not hold up the queues in `KEY` or `UNORDERED` modes, however in `PARTITION` mode it *will* block progress.
+Offsets will also continue to be committed (see <> and <>).
+
+=== Retry Delay Function
+
+As part of the https://github.com/confluentinc/parallel-consumer/issues/65[enhanced retry epic], the ability to https://github.com/confluentinc/parallel-consumer/issues/82[dynamically determine the retry delay] was added.
+This can be used to customise retry delay for a record, such as exponential back off or have different delays for different types of records, or have the delay determined by the status of a system etc.
+
+You can access the retry count of a record through it's wrapped `WorkContainer` class, which is the input variable to the retry delay function.
+
+.Example retry delay function implementing exponential backoff
+[source,java,indent=0]
+----
+ final double multiplier = 0.5;
+ final int baseDelaySecond = 1;
+
+ ParallelConsumerOptions.builder()
+ .retryDelayProvider(recordContext -> {
+ int numberOfFailedAttempts = recordContext.getNumberOfFailedAttempts();
+ long delayMillis = (long) (baseDelaySecond * Math.pow(multiplier, numberOfFailedAttempts) * 1000);
+ return Duration.ofMillis(delayMillis);
+ });
+----
+
+[[skipping-records]]
+=== Skipping Records
+
+If for whatever reason you want to skip a record, simply do not throw an exception, or catch any exception being thrown, log and swallow it and return from the user function normally.
+The system will treat this as a record processing success, mark the record as completed and move on as though it was a normal operation.
+
+A user may choose to skip a record for example, if it has been retried too many times or if the record is invalid or doesn't need processing.
+
+Implementing a https://github.com/confluentinc/parallel-consumer/issues/196[max retries feature] as a part of the system is planned.
+
+.Example of skipping a record after a maximum number of retries is reached
+[source,java,indent=0]
+----
+ final int maxRetries = 10;
+ final Map, Long> retriesCount = new ConcurrentHashMap<>();
+
+ pc.poll(context -> {
+ var consumerRecord = context.getSingleRecord().getConsumerRecord();
+ Long retryCount = retriesCount.computeIfAbsent(consumerRecord, ignore -> 0L);
+ if (retryCount < maxRetries) {
+ processRecord(consumerRecord);
+ // no exception, so completed - remove from map
+ retriesCount.remove(consumerRecord);
+ } else {
+ log.warn("Retry count {} exceeded max of {} for record {}", retryCount, maxRetries, consumerRecord);
+ // giving up, remove from map
+ retriesCount.remove(consumerRecord);
+ }
+ });
+----
+
+=== Circuit Breaker Pattern
+
+Although the system doesn't have an https://github.com/confluentinc/parallel-consumer/issues/110[explicit circuit breaker pattern feature], one can be created by combining the custom retry delay function and proactive failure.
+For example, the retry delay can be calculated based upon the status of an external system - i.e. if the external system is currently out of action, use a higher retry.
+Then in the processing function, again check the status of the external system first, and if it's still offline, throw an exception proactively without attempting to process the message.
+This will put the message back in the queue.
+
+.Example of circuit break implementation
+[source,java,indent=0]
+----
+ final Map upMap = new ConcurrentHashMap<>();
+
+ pc.poll(context -> {
+ var consumerRecord = context.getSingleRecord().getConsumerRecord();
+ String serverId = extractServerId(consumerRecord);
+ boolean up = upMap.computeIfAbsent(serverId, ignore -> true);
+
+ if (!up) {
+ up = updateStatusOfSever(serverId);
+ }
+
+ if (up) {
+ try {
+ processRecord(consumerRecord);
+ } catch (CircuitBreakingException e) {
+ log.warn("Server {} is circuitBroken, will retry message when server is up. Record: {}", serverId, consumerRecord);
+ upMap.put(serverId, false);
+ }
+ // no exception, so set server status UP
+ upMap.put(serverId, true);
+ } else {
+ throw new RuntimeException(msg("Server {} currently down, will retry record latter {}", up, consumerRecord));
+ }
+ });
+----
+
+=== Head of Line Blocking
+
+In order to have a failing record not block progress of a partition, one of the ordering modes other than `PARTITION` must be used, so that the system is allowed to process other messages that are perhaps in `KEY` order or in the case of `UNORDERED` processing - any message.
+This is because in `PARTITION` ordering mode, records are always processed in order of partition, and so the Head of Line blocking feature is effectively disabled.
+
+=== Future Work
+
+Improvements to this system are planned, see the following issues:
+
+* https://github.com/confluentinc/parallel-consumer/issues/65[Enhanced retry epic #65]
+* https://github.com/confluentinc/parallel-consumer/issues/48[Support scheduled message processing (scheduled retry)]
+* https://github.com/confluentinc/parallel-consumer/issues/196[Provide option for max retires, and a call back when reached (potential DLQ) #196]
+* https://github.com/confluentinc/parallel-consumer/issues/34[Monitor for progress and optionally shutdown (leave consumer group), skip message or send to DLQ #34]
+
+== Result Models
+
+* Void
+
+Processing is complete simply when your provided function finishes, and the offsets are committed.
+
+* Streaming User Results
+
+When your function is actually run, a result object will be streamed back to your client code, with information about the operation completion.
+
+* Streaming Message Publishing Results
+
+After your operation completes, you can also choose to publish a result message back to Kafka.
+The message publishing metadata can be streamed back to your client code.
+
+[[commit-mode]]
+== Commit Mode
+
+The system gives you three choices for how to do offset commits.
+The simplest of the three are the two Consumer commits modes.
+They are of course, `synchronous` and `asynchronous` mode.
+The `transactional` mode is explained in the next section.
+
+`Asynchronous` mode is faster, as it doesn't block the control loop.
+
+`Synchronous` will block the processing loop until a successful commit response is received, however, `Asynchronous` will still be capped by the max processing settings in the `ParallelConsumerOptions` class.
+
+If you're used to using the auto commit mode in the normal Kafka consumer, you can think of the `Asynchronous` mode being similar to this.
+We suggest starting with this mode, and it is the default.
+
+[[transaction-system]]
+=== Apache Kafka EoS Transaction Model in BULK
+
+There is also the option to use Kafka's Exactly Once Semantics (EoS) system.
+This causes all messages produced, by all workers in parallel, as a result of processing their messages, to be committed within a SINGLE, BULK transaction, along with their source offset.
+
+Note importantly - this is a BULK transaction, not a per input record transaction.
+
+This means that even under failure, the results will exist exactly once in the Kafka output topic.
+If as a part of your processing, you create side effects in other systems, this pertains to the usual idempotency requirements when breaking of EoS Kafka boundaries.
+
+CAUTION:: This is a BULK transaction, not a per input record transaction.
+There is not a single transaction per input record and per worker "thread", but one *LARGE* transaction that gets used by all parallel processing, until the commit interval.
+
+NOTE:: As with the `synchronous` processing mode, this will also block the processing loop until a successful transaction completes
+
+CAUTION: This cannot be true for any externally integrated third party system, unless that system is __idempotent__.
+
+For implementations details, see the <> section.
+
+.From the Options Javadoc
+[source,java,indent=0]
+----
+ /**
+ * Periodically commits through the Producer using transactions.
+ *
+ * Messages sent in parallel by different workers get added to the same transaction block - you end up with
+ * transactions 100ms (by default) "large", containing all records sent during that time period, from the
+ * offsets being committed.
+ *
+ * Of no use, if not also producing messages (i.e. using a {@link ParallelStreamProcessor#pollAndProduce}
+ * variation).
+ *
+ * Note: Records being sent by different threads will all be in a single transaction, as PC shares a single
+ * Producer instance. This could be seen as a performance overhead advantage, efficient resource use, in
+ * exchange for a loss in transaction granularity.
+ *
+ * The benefits of using this mode are:
+ *
+ * a) All records produced from a given source offset will either all be visible, or none will be
+ * ({@link org.apache.kafka.common.IsolationLevel#READ_COMMITTED}).
+ *
+ * b) If any records making up a transaction have a terminal issue being produced, or the system crashes before
+ * finishing sending all the records and committing, none will ever be visible and the system will eventually
+ * retry them in new transactions - potentially with different combinations of records from the original.
+ *
+ * c) A source offset, and it's produced records will be committed as an atomic set. Normally: either the record
+ * producing could fail, or the committing of the source offset could fail, as they are separate individual
+ * operations. When using Transactions, they are committed together - so if either operations fails, the
+ * transaction will never get committed, and upon recovery, the system will retry the set again (and no
+ * duplicates will be visible in the topic).
+ *
+ * This {@code CommitMode} is the slowest of the options, but there will be no duplicates in Kafka caused by
+ * producing a record multiple times if previous offset commits have failed or crashes have occurred (however
+ * message replay may cause duplicates in external systems which is unavoidable - external systems must be
+ * idempotent).
+ *
+ * The default commit interval {@link AbstractParallelEoSStreamProcessor#KAFKA_DEFAULT_AUTO_COMMIT_FREQUENCY}
+ * gets automatically reduced from the default of 5 seconds to 100ms (the same as Kafka Streams commit.interval.ms).
+ * Reducing this configuration places higher load on the broker, but will reduce (but cannot eliminate) replay
+ * upon failure. Note also that when using transactions in Kafka, consumption in {@code READ_COMMITTED} mode is
+ * blocked up to the offset of the first STILL open transaction. Using a smaller commit frequency reduces this
+ * minimum consumption latency - the faster transactions are closed, the faster the transaction content can be
+ * read by {@code READ_COMMITTED} consumers. More information about this can be found on the Confluent blog
+ * post:
+ * Enabling Exactly-Once in Kafka
+ * Streams.
+ *
+ * When producing multiple records (see {@link ParallelStreamProcessor#pollAndProduceMany}), all records must
+ * have been produced successfully to the broker before the transaction will commit, after which all will be
+ * visible together, or none.
+ *
+ * Records produced while running in this mode, won't be seen by consumer running in
+ * {@link ConsumerConfig#ISOLATION_LEVEL_CONFIG} {@link org.apache.kafka.common.IsolationLevel#READ_COMMITTED}
+ * mode until the transaction is complete and all records are produced successfully. Records produced into a
+ * transaction that gets aborted or timed out, will never be visible.
+ *
+ * The system must prevent records from being produced to the brokers whose source consumer record offsets has
+ * not been included in this transaction. Otherwise, the transactions would include produced records from
+ * consumer offsets which would only be committed in the NEXT transaction, which would break the EoS guarantees.
+ * To achieve this, first work processing and record producing is suspended (by acquiring the commit lock -
+ * see{@link #commitLockAcquisitionTimeout}, as record processing requires the produce lock), then succeeded
+ * consumer offsets are gathered, transaction commit is made, then when the transaction has finished, processing
+ * resumes by releasing the commit lock. This periodically slows down record production during this phase, by
+ * the time needed to commit the transaction.
+ *
+ * This is all separate from using an IDEMPOTENT Producer, which can be used, along with the
+ * {@link ParallelConsumerOptions#commitMode} {@link CommitMode#PERIODIC_CONSUMER_SYNC} or
+ * {@link CommitMode#PERIODIC_CONSUMER_ASYNCHRONOUS}.
+ *
+ * Failure:
+ *
+ * Commit lock: If the system cannot acquire the commit lock in time, it will shut down for whatever reason, the
+ * system will shut down (fail fast) - during the shutdown a final commit attempt will be made. The default
+ * timeout for acquisition is very high though - see {@link #commitLockAcquisitionTimeout}. This can be caused
+ * by the user processing function taking too long to complete.
+ *
+ * Produce lock: If the system cannot acquire the produce lock in time, it will fail the record processing and
+ * retry the record later. This can be caused by the controller taking too long to commit for some reason. See
+ * {@link #produceLockAcquisitionTimeout}. If using {@link #allowEagerProcessingDuringTransactionCommit}, this
+ * may cause side effect replay when the record is retried, otherwise there is no replay. See
+ * {@link #allowEagerProcessingDuringTransactionCommit} for more details.
+ *
+ * @see ParallelConsumerOptions.ParallelConsumerOptionsBuilder#commitInterval
+ */
+----
+
+[[streams-usage]]
+== Using with Kafka Streams
+
+Kafka Streams (KS) doesn't yet (https://cwiki.apache.org/confluence/display/KAFKA/KIP-311%3A+Async+processing+with+dynamic+scheduling+in+Kafka+Streams[KIP-311],
+https://cwiki.apache.org/confluence/display/KAFKA/KIP-408%3A+Add+Asynchronous+Processing+To+Kafka+Streams[KIP-408]) have parallel processing of messages.
+However, any given preprocessing can be done in KS, preparing the messages.
+One can then use this library to consume from an input topic, produced by KS to process the messages in parallel.
+
+For a code example, see the <> section.
+
+.Example usage with Kafka Streams
+image::https://lucid.app/publicSegments/view/43f2740c-2a7f-4b7f-909e-434a5bbe3fbf/image.png[Kafka Streams Usage,align="center"]
+[[mertics]]
+== Metrics
+
+Metrics collection subsystem is implemented using Micrometer. This allows for flexible configuration of target metrics backend to be used. See below on example of how to configure MeterRegistry for Parallel Consumer to use for metrics collection.
+
+=== Meters
+Following meters are defined by Parallel Consumer - grouped by Subsystem
+
+
+==== Partition Manager
+
+**Number Of Partitions**
+
+Gauge `pc.partitions.number{subsystem=partitions}`
+
+Number of partitions
+
+**Partition Incomplete Offsets**
+
+Gauge `pc.partition.incomplete.offsets{subsystem=partitions, topic="topicName", partition="partitionNumber"}`
+
+Number of incomplete offsets in the partition
+
+**Partition Highest Completed Offset**
+
+Gauge `pc.partition.highest.completed.offset{subsystem=partitions, topic="topicName", partition="partitionNumber"}`
+
+Highest completed offset in the partition
+
+**Partition Highest Sequential Succeeded Offset**
+
+Gauge `pc.partition.highest.sequential.succeeded.offset{subsystem=partitions, topic="topicName", partition="partitionNumber"}`
+
+Highest sequential succeeded offset in the partition
+
+**Partition Highest Seen Offset**
+
+Gauge `pc.partition.highest.seen.offset{subsystem=partitions, topic="topicName", partition="partitionNumber"}`
+
+Highest seen / consumed offset in the partition
+
+**Partition Last Committed Offset**
+
+Gauge `pc.partition.latest.committed.offset{subsystem=partitions, topic="topicName", partition="partitionNumber"}`
+
+Latest committed offset in the partition
+
+**Partition Assignment Epoch**
+
+Gauge `pc.partition.assignment.epoch{subsystem=partitions, topic="topicName", partition="partitionNumber"}`
+
+Epoch of partition assignment
+
+==== Processor
+
+**User Function Processing Time**
+
+Timer `pc.user.function.processing.time{subsystem=processor}`
+
+User function processing time
+
+**Dynamic Extra Load Factor**
+
+Gauge `pc.dynamic.load.factor{subsystem=processor}`
+
+Dynamic load factor - load of processing buffers
+
+**Pc Status**
+
+Gauge `pc.status{subsystem=processor}`
+
+PC Status, reported as number with following mapping - 0:UNUSED, 1:RUNNING, 2:PAUSED, 3:DRAINING, 4:CLOSING, 5:CLOSED
+
+==== Shard Manager
+
+**Number Of Shards**
+
+Gauge `pc.shards{subsystem=shardmanager}`
+
+Number of shards
+
+**Incomplete Offsets Total**
+
+Gauge `pc.incomplete.offsets.total{subsystem=shardmanager}`
+
+Total number of incomplete offsets
+
+**Shards Size**
+
+Gauge `pc.shards.size{subsystem=shardmanager}`
+
+Number of records queued for processing across all shards
+
+==== Work Manager
+
+**Inflight Records**
+
+Gauge `pc.inflight.records{subsystem=workmanager}`
+
+Total number of records currently being processed or waiting for retry
+
+**Waiting Records**
+
+Gauge `pc.waiting.records{subsystem=workmanager}`
+
+Total number of records waiting to be selected for processing
+
+**Processed Records**
+
+Counter `pc.processed.records{subsystem=workmanager, topic="topicName", partition="partitionNumber"}`
+
+Total number of records successfully processed
+
+**Failed Records**
+
+Counter `pc.failed.records{subsystem=workmanager, topic="topicName", partition="partitionNumber"}`
+
+Total number of records failed to be processed
+
+**Slow Records**
+
+Counter `pc.slow.records{subsystem=workmanager, topic="topicName", partition="partitionNumber"}`
+
+Total number of records that spent more than the configured time threshold in the waiting queue. This setting defaults to 10 seconds
+
+==== Broker Poller
+
+**Pc Poller Status**
+
+Gauge `pc.poller.status{subsystem=poller}`
+
+PC Broker Poller Status, reported as number with following mapping - 0:UNUSED, 1:RUNNING, 2:PAUSED, 3:DRAINING, 4:CLOSING, 5:CLOSED
+
+**Num Paused Partitions**
+
+Gauge `pc.partitions.paused{subsystem=poller}`
+
+Number of paused partitions
+
+==== Offset Encoder
+
+**Offsets Encoding Time**
+
+Timer `pc.offsets.encoding.time{subsystem=offsetencoder}`
+
+Time spend encoding offsets
+
+**Offsets Encoding Usage**
+
+Counter `pc.offsets.encoding.usage{subsystem=offsetencoder, codec="BitSet|BitSetCompressed|BitSetV2Compressed|RunLength"}`
+
+Offset encoding usage per encoding type
+
+**Metadata Space Used**
+
+Distribution Summary `pc.metadata.space.used{subsystem=offsetencoder}`
+
+Ratio between offset metadata payload size and available space
+
+**Payload Ratio Used**
+
+Distribution Summary `pc.payload.ratio.used{subsystem=offsetencoder}`
+
+Ratio between offset metadata payload size and offsets encoded
+
+=== Example Metrics setup steps
+Meter registry that metrics should be bound has to be set using Parallel Consumer Options along with any common tags that identify the PC instance.
+In addition, if desired - Kafka Consumer, Producer can be bound to the registry as well as general JVM metric, logging system and other common binders.
+
+Following example illustrates setup of Parallel Consumer with Meter Registry and binds Kafka Consumer to that same registry as well.
+
+[source,java,indent=0]
+----
+ ParallelStreamProcessor setupParallelConsumer() {
+ Consumer kafkaConsumer = getKafkaConsumer();
+ String instanceId = UUID.randomUUID().toString();
+ var options = ParallelConsumerOptions.builder()
+ .ordering(ParallelConsumerOptions.ProcessingOrder.KEY)
+ .maxConcurrency(1000)
+ .consumer(kafkaConsumer)
+ .meterRegistry(meterRegistry) //<1>
+ .metricsTags(Tags.of(Tag.of("common-tag", "tag1"))) //<2>
+ .pcInstanceTag(instanceId) //<3>
+ .build();
+
+ ParallelStreamProcessor eosStreamProcessor =
+ ParallelStreamProcessor.createEosStreamProcessor(options);
+
+ eosStreamProcessor.subscribe(of(inputTopic));
+
+ kafkaClientMetrics = new KafkaClientMetrics(kafkaConsumer); //<4>
+ kafkaClientMetrics.bindTo(meterRegistry); //<5>
+ return eosStreamProcessor;
+ }
+----
+<1> - Meter Registry is set through ParallelConsumerOptions.builder(), if not specified - will default to CompositeMeterRegistry - which is No-op.
+<2> - Optional - common tags can be specified through same builder - they will be added to all Parallel Consumer meters
+<3> - Optional - instance tag value can be specified - it has to be unique to ensure meter uniqueness in cases when multiple parallel consumer instances are recording metrics to the same meter registry. If instance tag is not specified - unique UUID value will be generated and used. Tag is created with tag key 'pcinstance'.
+<4> - Optional - Kafka Consumer Micrometer metrics object created for Kafka Consumer that is later used for Parallel Consumer.
+<5> - Optional - Kafka Consumer Micrometer metrics are bound to Meter Registry.
+
+NOTE:: any additional binders / metrics need to be cleaned up appropriately - for example the Kafka Consumer Metrics registered above - need to be closed using `kafkaClientMetrics.close()` after calling shutting down Parallel Consumer as Parallel Consumer will close Kafka Consumer on shutdown.
+
+
+[[roadmap]]
+== Roadmap
+
+For released changes, see the link:CHANGELOG.adoc[CHANGELOG].
+
+For features in development and a more accurate view on the roadmap, have a look at the
+https://github.com/confluentinc/parallel-consumer/issues[GitHub issues], and clone https://github.com/astubbs/parallel-consumer[Antony's fork].
+
+== Usage Requirements
+
+* Client side
+** JDK 8
+** SLF4J
+** Apache Kafka (AK) Client libraries 2.5
+** Supports all features of the AK client (e.g. security setups, schema registry etc)
+** For use with Streams, see <> section
+** For use with Connect:
+*** Source: simply consume from the topic that your Connect plugin is publishing to
+*** Sink: use the poll and producer style API and publish the records to the topic that the connector is sinking from
+* Server side
+** Should work with any cluster that the linked AK client library works with
+*** If using EoS/Transactions, needs a cluster setup that supports EoS/transactions
+
+== Development Information
+
+=== Requirements
+
+* Uses https://projectlombok.org/setup/intellij[Lombok], if you're using IntelliJ Idea, get the https://plugins.jetbrains.com/plugin/6317-lombok[plugin].
+* Integration tests require a https://docs.docker.com/docker-for-mac/[running locally accessible Docker host].
+* Has a Maven `profile` setup for IntelliJ Idea, but not Eclipse for example.
+
+=== Notes
+
+The unit test code is set to run at a very high frequency, which can make it difficult to read debug logs (or impossible).
+If you want to debug the code or view the main logs, consider changing the below:
+
+// replace with code inclusion from readme branch
+.ParallelEoSStreamProcessorTestBase
+[source]
+----
+ParallelEoSStreamProcessorTestBase#DEFAULT_BROKER_POLL_FREQUENCY_MS
+ParallelEoSStreamProcessorTestBase#DEFAULT_COMMIT_INTERVAL_MAX_MS
+----
+
+=== Recommended IDEA Plugins
+
+* AsciiDoc
+* CheckStyle
+* CodeGlance
+* EditorConfig
+* Rainbow Brackets
+* SonarLint
+* Lombok
+
+=== Readme
+
+The `README` uses a special https://github.com/whelk-io/asciidoc-template-maven-plugin/pull/25[custom maven processor plugin] to import live code blocks into the root readme, so that GitHub can show the real code as includes in the `README`.
+This is because GitHub https://github.com/github/markup/issues/1095[doesn't properly support the _include_ directive].
+
+The source of truth readme is in link:{project_root}/src/docs/README_TEMPLATE.adoc[].
+
+=== Maven targets
+
+[qanda]
+Compile and run all tests::
+`mvn verify`
+
+Run tests excluding the integration tests::
+`mvn test`
+
+Run all tests::
+`mvn verify`
+
+Run any goal skipping tests (replace `` e.g. `install`)::
+`mvn -DskipTests`
+
+See what profiles are active::
+`mvn help:active-profiles`
+
+See what plugins or dependencies are available to be updated::
+`mvn versions:display-plugin-updates versions:display-property-updates versions:display-dependency-updates`
+
+Run a single unit test::
+`mvn -Dtest=TestCircle test`
+
+Run a specific integration test method in a submodule project, skipping unit tests::
+`mvn -Dit.test=TransactionAndCommitModeTest#testLowMaxPoll -DskipUTs=true verify -DfailIfNoTests=false --projects parallel-consumer-core`
+
+Run `git bisect` to find a bad commit, edit the Maven command in `bisect.sh` and run::
+
+[source=bash]
+----
+git bisect start good bad
+git bisect run ./bisect.sh
+----
+
+Note::
+`mvn compile` - Due to a bug in Maven's handling of test-jar dependencies - running `mvn compile` fails, use `mvn test-compile` instead.
+See https://github.com/confluentinc/parallel-consumer/issues/162[issue #162]
+and this https://stackoverflow.com/questions/4786881/why-is-test-jar-dependency-required-for-mvn-compile[Stack Overflow question].
+
+=== Testing
+
+The project has good automated test coverage, of all features.
+Including integration tests running against real Kafka broker and database.
+If you want to run the tests yourself, clone the repository and run the command: `mvn test`.
+The tests require an active docker server on `localhost`.
+
+==== Integration Testing with TestContainers
+//https://github.com/confluentinc/schroedinger#integration-testing-with-testcontainers
+
+We use the excellent https://testcontainers.org[Testcontainers] library for integration testing with JUnit.
+
+To speed up test execution, you can enable container reuse across test runs by setting the following in your https://www.testcontainers.org/features/configuration/[`~/.testcontainers.properties` file]:
+
+[source]
+----
+testcontainers.reuse.enable=true
+----
+
+This will leave the container running after the JUnit test is complete for reuse by subsequent runs.
+
+> NOTE: The container will only be left running if it is not explicitly stopped by the JUnit rule.
+> For this reason, we use a variant of the https://www.testcontainers.org/test_framework_integration/manual_lifecycle_control/#singleton-containers[singleton container pattern]
+> instead of the JUnit rule.
+
+Testcontainers detects if a container is reusable by hashing the container creation parameters from the JUnit test.
+If an existing container is _not_ reusable, a new container will be created, **but the old container will not be removed**.
+
+Target | Description --- | ---
+`testcontainers-list` | List all containers labeled as testcontainers
+`testcontainers-clean` | Remove all containers labeled as testcontainers
+
+.Stop and remove all containers labeled with `org.testcontainers=true`
+[source,bash]
+----
+docker container ls --filter 'label=org.testcontainers=true' --format '{{.ID}}' \
+| $(XARGS) docker container rm --force
+----
+
+.List all containers labeled with `org.testcontainers=true`
+[source,bash]
+----
+docker container ls --filter 'label=org.testcontainers=true'
+----
+
+> NOTE: `testcontainers-clean` removes **all** docker containers on your system with the `io.testcontainers=true` label > (including the most recent container which may be reusable).
+
+See https://github.com/testcontainers/testcontainers-java/pull/1781[this testcontainers PR] for details on the reusable containers feature.
+
+== Implementation Details
+
+=== Core Architecture
+
+Concurrency is controlled by the size of the thread pool (`worker pool` in the diagram).
+Work is performed in a blocking manner, by the users submitted lambda functions.
+
+These are the main sub systems:
+
+- controller thread
+- broker poller thread
+- work pool thread
+- work management
+- offset map manipulation
+
+Each thread collaborates with the others through thread safe Java collections.
+
+.Core Architecture. Threads are represented by letters and colours, with their steps in sequential numbers.
+image::https://lucid.app/publicSegments/view/320d924a-6517-4c54-a72e-b1c4b22e59ed/image.png[Core Architecture,align="center"]
+
+=== Vert.x Architecture
+
+The Vert.x module is an optional extension to the core module.
+As depicted in the diagram, the architecture extends the core architecture.
+
+Instead of the work thread pool count being the degree of concurrency, it is controlled by a max parallel requests setting, and work is performed asynchronously on the Vert.x engine by a _core_ count aligned Vert.x managed thread pool using Vert.x asynchronous IO plugins (https://vertx.io/docs/vertx-core/java/#_verticles[verticles]).
+
+.Vert.x Architecture
+image::https://lucid.app/publicSegments/view/509df410-5997-46be-98e7-ac7f241780b4/image.png[Vert.x Architecture,align="center"]
+
+=== Transactional System Architecture
+
+image::https://lucid.app/publicSegments/view/7480d948-ed7d-4370-a308-8ec12e6b453b/image.png[]
+
+[[offset_map]]
+=== Offset Map
+
+Unlike a traditional queue, messages are not deleted on an acknowledgement.
+However, offsets *are* tracked *per message*, per consumer group - there is no message replay for successful messages, even over clean restarts.
+
+Across a system failure, only completed messages not stored as such in the last offset payload commit will be replayed.
+This is not an _exactly once guarantee_, as message replay cannot be prevented across failure.
+
+CAUTION: Note that Kafka's Exactly Once Semantics (EoS) (transactional processing) also does not prevent _duplicate message replay_ - it *presents* an _effectively once_ result messages in Kafka topics.
+Messages may _still_ be replayed when using `EoS`.
+This is an important consideration when using it, especially when integrating with thrid party systems, which is a very common pattern for utilising this project.
+
+As mentioned previously, offsets are always committed in the correct order and only once all previous messages have been successfully processed; regardless of <> selected.
+We call this the "highest committable offset".
+
+However, because messages can be processed out of order, messages beyond the highest committable offset must also be tracked for success and not replayed upon restart of failure.
+To achieve this the system goes a step further than normal Kafka offset commits.
+
+When messages beyond the highest committable offset are successfully processed;
+
+. they are stored as such in an internal memory map.
+. when the system then next commits offsets
+. if there are any messages beyond the highest offset which have been marked as succeeded
+.. the offset map is serialised and encoded into a base 64 string, and added to the commit message metadata.
+. upon restore, if needed, the system then deserializes this offset map and loads it back into memory
+. when each messages is polled into the system
+.. it checks if it's already been previously completed
+.. at which point it is then skipped.
+
+This ensures that no message is reprocessed if it's been previously completed.
+
+IMPORTANT: Successful messages beyond the _highest committable offset_ are still recorded as such in a specially constructed metadata payload stored alongside the Kafka committed offset.
+These messages are not replayed upon restore/restart.
+
+The offset map is compressed in parallel using two different compression techniques - run length encoding and bitmap encoding.
+The sizes of the compressed maps are then compared, and the smallest chosen for serialization.
+If both serialised formats are significantly large, they are then both compressed using `zstd` compression, and if that results in a smaller serialization then the compressed form is used instead.
+
+
+==== Storage Notes
+
+* Runtime data model creates list of incomplete offsets
+* Continuously builds a full complete / not complete bit map from the base offset to be committed
+* Dynamically switching storage
+** encodes into a `BitSet`, and a `RunLength`, then compresses both using zstd, then uses the smallest and tags as such in the encoded String
+** Which is smallest can depend on the size and information density of the offset map
+*** Smaller maps fit better into uncompressed `BitSets` ~(30 entry map bitset: compressed: 13 Bytes, uncompressed: 4 Bytes)
+*** Larger maps with continuous sections usually better in compressed `RunLength`
+*** Completely random offset maps, compressed and uncompressed `BitSet` is roughly the same (2000 entries, uncompressed bitset: 250, compressed: 259, compressed bytes array: 477)
+*** Very large maps (20,000 entries), a compressed `BitSet` seems to be significantly smaller again if random.
+* Gets stored along with base offset for each partition, in the offset `commitsync` `metadata` string
+* The offset commit metadata has a hardcoded limit of 4096 bytes (4 kb) per partition (@see `kafka.coordinator.group.OffsetConfig#DefaultMaxMetadataSize = 4096`)
+** Because of this, if our map doesn't fit into this, we have to drop it and not use it, losing the shorter replay benefits.
+However, with runlength encoding and typical offset patterns this should be quite rare.
+*** Work is being done on continuous and predictive space requirements, which will optionally prevent the system from continuing past a point by introducing local backpressure which it can't proceed without dropping the encoded map information - see https://github.com/confluentinc/parallel-consumer/issues/53[Exact continuous offset encoding for precise offset payload size back pressure].
+** Not being able to fit the map into the metadata, depends on message acknowledgement patterns in the use case and the numbers of messages involved.
+Also, the information density in the map (i.e. a single not yet completed message in 4000 completed ones will be a tiny map and will fit very large amounts of messages)
+
+===== FAQ
+
+[qanda]
+If for example, offset 5 cannot be processed for whatever reason, does it cause the committed offset to stick to 5?::
+Yes - the committed offset would "stick" to 5, with the metadata payload containing all the per msg ack's beyond 5.
++
+(Reference: https://github.com/confluentinc/parallel-consumer/issues/415#issuecomment-1256022394[#415])
+
+In the above scenario, would the system eventually exceed the OffsetMap size limit?::
+No, as if the payload size hits 75% or more of the limit (4kB), the back pressure system kicks in, and no more records will be taken for processing, until it drops below 75% again.
+Instead, it will keep retrying existing records.
++
+However, note that if the only record to continually fail is 5, and all others succeed, let's say offset 6-50,000, then the metadata payload is only ~2 shorts (1 and (50,000-6=) 49,994), as it will use run length encoding.
+So it's very efficient.
++
+(Reference: https://github.com/confluentinc/parallel-consumer/issues/415#issuecomment-1256022394[#415])
+
+== Attribution
+
+http://www.apache.org/[Apache®], http://kafka.apache.org/[Apache Kafka], and http://kafka.apache.org/[Kafka®] are either registered trademarks or trademarks of the http://www.apache.org/[Apache Software Foundation] in the United States and/or other countries.
+
+== Tools
+
+image:https://www.yourkit.com/images/yklogo.png[link=https://www.yourkit.com/java/profiler/index.jsp,YourKit]
+
+Quite simply the best profiler for Java, and the only one I use.
+I have been using it for decades.
+Quick, easy to use but soo powerful.
+
+YourKit supports open source projects with innovative and intelligent tools for monitoring and profiling Java and .NET applications.
+
+YourKit is the creator of https://www.google.com/url?q=https://www.yourkit.com/java/profiler/&source=gmail-imap&ust=1670918364000000&usg=AOvVaw3kaQak_H7lmT_plCEzxvde[YourKit Java Profiler],
+https://www.google.com/url?q=https://www.yourkit.com/.net/profiler/&source=gmail-imap&ust=1670918364000000&usg=AOvVaw1ZgQhyH2rIOHTuqtTjFAsA[YourKit .NET Profiler], and https://www.google.com/url?q=https://www.yourkit.com/youmonitor/&source=gmail-imap&ust=1670918364000000&usg=AOvVaw13UzOhGkJLEn-Md3-GNjYB[YourKit YouMonitor].
+
+:leveloffset: +1
+:toc: macro
+:toclevels: 1
+
+= Change Log
+
+A high level summary of noteworthy changes in each version.
+
+NOTE:: Dependency version bumps are not listed here.
+
+// git log --pretty="* %s" 0.3.0.2..HEAD
+
+// only show TOC if this is the root document (not in the README)
+ifndef::github_name[]
+toc::[]
+endif::[]
+== 0.5.2.8
+
+=== Fixes
+
+* fix: Fix equality and hash code for ShardKey with array key (#638), resolves (#579)
+
+== 0.5.2.7
+
+=== Fixes
+
+* fix: Return cached pausedPartitionSet (#620), resolves (#618)
+* fix: Parallel consumer stops processing data sometimes (#623), fixes (#606)
+* fix: Add synchronization to ensure proper intializaiton and closing of PCMetrics singleton (#627), fixes (#617)
+* fix: Readme - metrics example correction (#614)
+* fix: Remove micrometer-atlas dependency (#628), fixes (#625)
+
+=== Improvements
+
+* Refactored metrics implementation to not use singleton - improves meter separation, allows correct metrics subsystem operation when multiple parallel consumer instances are running in same java process (#630), fixes (#617) improves on (#627)
+
+== 0.5.2.6
+=== Improvements
+
+* feature: Micrometer metrics (#594)
+* feature: Adds an option to pass an invalid offset metadata error policy (#537), improves (#326)
+* feature: Lazy intialization of workerThreadPool (#531)
+
+=== Fixes
+
+* fix: Don't drain mode shutdown kills inflight threads (#559)
+* fix: Drain mode shutdown doesn't pause consumption correctly (#552)
+* fix: RunLength offset decoding returns 0 base offset after no-progress commit - related to (#546)
+* fix: Transactional PConsumer stuck while rebalancing - related to (#541)
+
+=== Dependencies
+
+* PL-211: Update dependencies from dependabot, Add mvnw, use mvnw in jenkins (#583)
+* PL-211: Update dependencies from dependabot (#589)
+
+== 0.5.2.5
+
+=== Fixes
+
+* fixes: #195 NoSuchFieldException when using consumer inherited from KafkaConsumer (#469)
+* fix: After new performance fix PR#530 merges - corner case could cause out of order processing (#534)
+* fix: Cleanup WorkManager's count of in-progress work, when work is stale after partition revocation (#547)
+
+=== Improvements
+
+* perf: Adds a caching layer to work management to alleviate O(n) counting (#530)
+
+== 0.5.2.4
+
+=== Improvements
+
+* feature: Simple PCRetriableException to remove error spam from logs (#444)
+* minor: fixes #486: Missing generics in JStreamParallelStreamProcessor #491
+* minor: partially address #459: Moves isClosedOrFailed into top level ParallelConsumer interface (#491)
+* tests: Demonstrates how to use MockConsumer with PC for issue #176
+* other minor improvements
+
+=== Fixes
+
+* fixes #409: Adds support for compacted topics and commit offset resetting (#425)
+** Truncate the offset state when bootstrap polled offset higher or lower than committed
+** Prune missing records from the tracked incomplete offset state, when they're missing from polled batches
+* fix: Improvements to encoding ranges (int vs long) #439
+** Replace integer offset references with long - use Long everywhere we deal with offsets, and where we truncate down, do it exactly, detect and handle truncation issues.
+
+== 0.5.2.3
+
+=== Improvements
+
+* Transactional commit mode system improvements and docs (#355)
+** Clarifies transaction system with much better documentation.
+** Fixes a potential race condition which could cause offset leaks between transactions boundaries.
+** Introduces lock acquisition timeouts.
+** Fixes a potential issue with removing records from the retry queue incorrectly, by having an inconsistency between compareTo and equals in the retry TreeMap.
+* Adds a very simple Dependency Injection system modeled on Dagger (#398)
+* Various refactorings e.g. new ProducerWrap
+
+* Dependencies
+** build(deps): prod: zstd, reactor, dev: podam, progressbar, postgresql maven-plugins: versions, help (#420)
+** build(deps-dev): bump postgresql from 42.4.1 to 42.5.0
+** bump podam, progressbar, zstd, reactor
+** build(deps): bump versions-maven-plugin from 2.11.0 to 2.12.0
+** build(deps): bump maven-help-plugin from 3.2.0 to 3.3.0
+** build(deps-dev): bump Confluent Platform Kafka Broker to 7.2.2 (#421)
+** build(deps): Upgrade to AK 3.3.0 (#309)
+
+
+=== Fixes
+
+* fixes #419: NoSuchElementException during race condition in PartitionState (#422)
+* Fixes #412: ClassCastException with retryDelayProvider (#417)
+* fixes ShardManager retryQueue ordering and set issues due to poor Comparator implementation (#423)
+
+
+== v0.5.2.2
+
+=== Fixes
+
+- Fixes dependency scope for Mockito from compile to test (#376)
+
+== v0.5.2.1
+
+=== Fixes
+
+- Fixes regression issue with order of state truncation vs commit (#362)
+
+== v0.5.2.0
+
+=== Fixes and Improvements
+
+- fixes #184: Fix multi topic subscription with KEY order by adding topic to shard key (#315)
+- fixes #329: Committing around transaction markers causes encoder to crash (#328)
+- build: Upgrade Truth-Generator to 0.1.1 for user Subject discovery (#332)
+
+=== Build
+
+- build: Allow snapshots locally, fail in CI (#331)
+- build: OSS Index scan change to warn only and exclude Guava CVE-2020-8908 as it's WONT_FIX (#330)
+
+=== Dependencies
+
+- build(deps): bump reactor-core from 3.4.19 to 3.4.21 (#344)
+- build(deps): dependabot bump Mockito, Surefire, Reactor, AssertJ, Release (#342) (#342)
+- build(deps): dependabot bump TestContainers, Vert.x, Enforcer, Versions, JUnit, Postgress (#336)
+
+=== Linked issues
+
+- Message with null key lead to continuous failure when using KEY ordering #318
+- Subscribing to two or more topics with KEY ordering, results in messages of the same Key never being processed #184
+- Cannot have negative length BitSet error - committing transaction adjacent offsets #329
+
+== v0.5.1.0
+
+=== Features
+
+* #193: Pause / Resume PC (circuit breaker) without unsubscribing from topics
+
+=== Fixes and Improvements
+
+* #225: Build and runtime support for Java 16+ (#289)
+* #306: Change Truth-Generator dependency from compile to test
+* #298: Improve PollAndProduce performance by first producing all records, and then waiting for the produce results.Previously, this was done for each ProduceRecord individually.
+
+== v0.5.0.0
+
+=== Features
+
+* feature: Poll Context object for API (#223)
+** PollContext API - provides central access to result set with various convenience methods as well as metadata about records, such as failure count
+* major: Batching feature and Event system improvements
+** Batching - all API methods now support batching.
+See the Options class set batch size for more information.
+
+=== Fixes and Improvements
+
+* Event system - better CPU usage in control thread
+* Concurrency stability improvements
+* Update dependencies
+* #247: Adopt Truth-Generator (#249)
+** Adopt https://github.com/astubbs/truth-generator[Truth Generator] for automatic generation of https://truth.dev/[Google Truth] Subjects
+* Large rewrite of internal architecture for improved maintence and simplicity which fixed some corner case issues
+** refactor: Rename PartitionMonitor to PartitionStateManager (#269)
+** refactor: Queue unification (#219)
+** refactor: Partition state tracking instead of search (#218)
+** refactor: Processing Shard object
+* fix: Concurrency and State improvements (#190)
+
+=== Build
+
+* build: Lock TruthGenerator to 0.1 (#272)
+* build: Deploy SNAPSHOTS to maven central snaphots repo (#265)
+* build: Update Kafka to 3.1.0 (#229)
+* build: Crank up Enforcer rules and turn on ossindex audit
+* build: Fix logback dependency back to stable
+* build: Upgrade TestContainer and CP
+
+== v0.4.0.1
+
+=== Improvements
+
+- Add option to specify timeout for how long to wait offset commits in periodic-consumer-sync commit-mode
+- Add option to specify timeout for how long to wait for blocking Producer#send
+
+=== Docs
+
+- docs: Confluent Cloud configuration links
+- docs: Add Confluent's product page for PC to README
+- docs: Add head of line blocking to README
+
+== v0.4.0.0
+// https://github.com/confluentinc/parallel-consumer/releases/tag/0.4.0.0
+
+=== Features
+
+* https://projectreactor.io/[Project Reactor] non-blocking threading adapter module
+* Generic Vert.x Future support - i.e. FileSystem, db etc...
+
+=== Fixes and Improvements
+
+* Vert.x concurrency control via WebClient host limits fixed - see #maxCurrency
+* Vert.x API cleanup of invalid usage
+* Out of bounds for empty collections
+* Use ConcurrentSkipListMap instead of TreeMap to prevent concurrency issues under high pressure
+* log: Show record topic in slow-work warning message
+
+== v0.3.2.0
+
+=== Fixes and Improvements
+
+* Major: Upgrade to Apache Kafka 2.8 (still compatible with 2.6 and 2.7 though)
+* Adds support for managed executor service (Java EE Compatibility feature)
+* #65 support for custom retry delay providers
+
+== v0.3.1.0
+
+=== Fixes and Improvements
+
+* Major refactor to code base - primarily the two large God classes
+** Partition state now tracked separately
+** Code moved into packages
+* Busy spin in some cases fixed (lower CPU usage)
+* Reduce use of static data for test assertions - remaining identified for later removal
+* Various fixes for parallel testing stability
+
+== v0.3.0.3
+
+=== Fixes and Improvements
+
+==== Overview
+
+* Tests now run in parallel
+* License fixing / updating and code formatting
+* License format runs properly now when local, check on CI
+* Fix running on Windows and Linux
+* Fix JAVA_HOME issues
+
+==== Details:
+
+* tests: Enable the fail fast feature now that it's merged upstream
+* tests: Turn on parallel test runs
+* format: Format license, fix placement
+* format: Apply Idea formatting (fix license layout)
+* format: Update mycila license-plugin
+* test: Disable redundant vert.x test - too complicated to fix for little gain
+* test: Fix thread counting test by closing PC @After
+* test: Test bug due to static state overrides when run as a suite
+* format: Apply license format and run every All Idea build
+* format: Organise imports
+* fix: Apply license format when in dev laptops - CI only checks
+* fix: javadoc command for various OS and envs when JAVA_HOME missing
+* fix: By default, correctly run time JVM as jvm.location
+
+== v0.3.0.2
+
+=== Fixes and Improvements
+
+* ci: Add CODEOWNER
+* fix: #101 Validate GroupId is configured on managed consumer
+* Use 8B1DA6120C2BF624 GPG Key For Signing
+* ci: Bump jdk8 version path
+* fix: #97 Vert.x thread and connection pools setup incorrect
+* Disable Travis and Codecov
+* ci: Apache Kafka and JDK build matrix
+* fix: Set Serdes for MockProducer for AK 2.7 partition fix KAFKA-10503 to fix new NPE
+* Only log slow message warnings periodically, once per sweep
+* Upgrade Kafka container version to 6.0.2
+* Clean up stalled message warning logs
+* Reduce log-level if no results are returned from user-function (warn -> debug)
+* Enable java 8 Github
+* Fixes #87 - Upgrade UniJ version for UnsupportedClassVersion error
+* Bump TestContainers to stable release to specifically fix #3574
+* Clarify offset management capabilities
+
+== v0.3.0.1
+
+* fixes #62: Off by one error when restoring offsets when no offsets are encoded in metadata
+* fix: Actually skip work that is found as stale
+
+== v0.3.0.0
+
+=== Features
+
+* Queueing and pressure system now self tuning, performance over default old tuning values (`softMaxNumberMessagesBeyondBaseCommitOffset` and `maxMessagesToQueue`) has doubled.
+** These options have been removed from the system.
+* Offset payload encoding back pressure system
+** If the payload begins to take more than a certain threshold amount of the maximum available, no more messages will be brought in for processing, until the space need beings to reduce back below the threshold.
+This is to try to prevent the situation where the payload is too large to fit at all, and must be dropped entirely.
+** See Proper offset encoding back pressure system so that offset payloads can't ever be too large https://github.com/confluentinc/parallel-consumer/issues/47[#47]
+** Messages that have failed to process, will always be allowed to retry, in order to reduce this pressure.
+
+=== Improvements
+
+* Default ordering mode is now `KEY` ordering (was `UNORDERED`).
+** This is a better default as it's the safest mode yet high performing mode.
+It maintains the partition ordering characteristic that all keys are processed in log order, yet for most use cases will be close to as fast as `UNORDERED` when the key space is large enough.
+* https://github.com/confluentinc/parallel-consumer/issues/37[Support BitSet encoding lengths longer than Short.MAX_VALUE #37] - adds new serialisation formats that supports wider range of offsets - (32,767 vs 2,147,483,647) for both BitSet and run-length encoding.
+* Commit modes have been renamed to make it clearer that they are periodic, not per message.
+* Minor performance improvement, switching away from concurrent collections.
+
+=== Fixes
+
+* Maximum offset payload space increased to correctly not be inversely proportional to assigned partition quantity.
+* Run-length encoding now supports compacted topics, plus other bug fixes as well as fixes to Bitset encoding.
+
+== v0.2.0.3
+
+=== Fixes
+
+** https://github.com/confluentinc/parallel-consumer/issues/35[Bitset overflow check (#35)] - gracefully drop BitSet or Runlength encoding as an option if offset difference too large (short overflow)
+*** A new serialisation format will be added in next version - see https://github.com/confluentinc/parallel-consumer/issues/37[Support BitSet encoding lengths longer than Short.MAX_VALUE #37]
+** Gracefully drops encoding attempts if they can't be run
+** Fixes a bug in the offset drop if it can't fit in the offset metadata payload
+
+== v0.2.0.2
+
+=== Fixes
+
+** Turns back on the https://github.com/confluentinc/parallel-consumer/issues/35[Bitset overflow check (#35)]
+
+== v0.2.0.1 DO NOT USE - has critical bug
+
+=== Fixes
+
+** Incorrectly turns off an over-flow check in https://github.com/confluentinc/parallel-consumer/issues/35[offset serialisation system (#35)]
+
+== v0.2.0.0
+
+=== Features
+
+** Choice of commit modes: Consumer Asynchronous, Synchronous and Producer Transactions
+** Producer instance is now optional
+** Using a _transactional_ Producer is now optional
+** Use the Kafka Consumer to commit `offsets` Synchronously or Asynchronously
+
+=== Improvements
+
+** Memory performance - garbage collect empty shards when in KEY ordering mode
+** Select tests adapted to non transactional (multiple commit modes) as well
+** Adds supervision to broker poller
+** Fixes a performance issue with the async committer not being woken up
+** Make committer thread revoke partitions and commit
+** Have onPartitionsRevoked be responsible for committing on close, instead of an explicit call to commit by controller
+** Make sure Broker Poller now drains properly, committing any waiting work
+
+=== Fixes
+
+** Fixes bug in commit linger, remove genesis offset (0) from testing (avoid races), add ability to request commit
+** Fixes #25 https://github.com/confluentinc/parallel-consumer/issues/25:
+*** Sometimes a transaction error occurs - Cannot call send in state COMMITTING_TRANSACTION #25
+** ReentrantReadWrite lock protects non-thread safe transactional producer from incorrect multithreaded use
+** Wider lock to prevent transaction's containing produced messages that they shouldn't
+** Must start tx in MockProducer as well
+** Fixes example app tests - incorrectly testing wrong thing and MockProducer not configured to auto complete
+** Add missing revoke flow to MockConsumer wrapper
+** Add missing latch timeout check
+
+== v0.1
+
+=== Features:
+
+** Have massively parallel consumption processing without running hundreds or thousands of
+*** Kafka consumer clients
+*** topic partitions
++
+without operational burden or harming the clusters performance
+** Efficient individual message acknowledgement system (without local or third system state) to massively reduce message replay upon failure
+** Per `key` concurrent processing, per `partition` and unordered message processing
+** `Offsets` committed correctly, in order, of only processed messages, regardless of concurrency level or retries
+** Vert.x non-blocking library integration (HTTP currently)
+** Fair partition traversal
+** Zero~ dependencies (`Slf4j` and `Lombok`) for the core module
+** Java 8 compatibility
+** Throttle control and broker liveliness management
+** Clean draining shutdown cycle
+//:leveloffset: -1 - Duplicate key leveloffset (attempted merging values +1 and -1): https://github.com/whelk-io/asciidoc-template-maven-plugin/issues/118
+
diff --git a/RELEASE.adoc b/RELEASE.adoc
new file mode 100644
index 000000000..65994e35d
--- /dev/null
+++ b/RELEASE.adoc
@@ -0,0 +1,15 @@
+= Releasing
+
+- Update the changelog and commit
+- Run the maven release:prepare goal:
+
+`release:prepare -DautoVersionSubmodules=true -DpushChanges=false -Darguments=-DskipTests -Pci`
+
+- Push the master branch with release and tag
+- Trigger master builder to build the tag (this is needed to trigger the deployment flow)
+- Wait for Jenkins to finish running the build (~15 minutes)
+- Wait for Sonatype to publish from it's staging area (~15 minutes) https://repo1.maven.org/maven2/io/confluent/parallelconsumer/parallel-consumer-parent/[repo1 link]
+- Verify the release is available on Maven Central https://repo1.maven.org/maven2/io/confluent/parallelconsumer/parallel-consumer-parent/[repo1 link]
+- Create the release on GH from the tag
+- Paste in the details from the changelog, save, share as discussion
+- Announce on slack (community #clients and internal channels), mailing list, twitter
\ No newline at end of file
diff --git a/bin/build-parallel-consumer-core-without-tests.sh b/bin/build-parallel-consumer-core-without-tests.sh
new file mode 100755
index 000000000..bff424860
--- /dev/null
+++ b/bin/build-parallel-consumer-core-without-tests.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2020-2022 Confluent, Inc.
+#
+
+export JAVA_HOME=$(/usr/libexec/java_home -v13)
+mvn clean install -pl parallel-consumer-core -Dmaven.test.skip=true
diff --git a/bin/build-without-tests.sh b/bin/build-without-tests.sh
new file mode 100755
index 000000000..5fbf92da4
--- /dev/null
+++ b/bin/build-without-tests.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2020-2022 Confluent, Inc.
+#
+
+export JAVA_HOME=$(/usr/libexec/java_home -v13)
+mvn clean install -Dmaven.test.skip=true
diff --git a/bin/checkcompile-license.sh b/bin/checkcompile-license.sh
new file mode 100644
index 000000000..90984f906
--- /dev/null
+++ b/bin/checkcompile-license.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Copyright (C) 2020-2022 Confluent, Inc.
+#
+
+
+COMMITS=$(git log --oneline HEAD...parralel-test-fix^ | cut -d " " -f 1)
+
+testCommit() {
+ # COMMIT = $1
+ echo Checking out commit $COMMIT
+ git checkout $COMMIT >/dev/null 2>/dev/null
+
+ # mvn compile test-compile > /dev/null 2> /dev/null
+ mvn license:check
+
+ if [ $? -eq 0 ]; then
+ echo $COMMIT passed
+ else
+ echo $COMMIT failed
+ fi
+}
+
+for COMMIT in $COMMITS; do
+ testCommit "$COMMIT"
+done
diff --git a/bin/deploy.sh b/bin/deploy.sh
new file mode 100755
index 000000000..52590b3a9
--- /dev/null
+++ b/bin/deploy.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2020-2022 Confluent, Inc.
+#
+
+export JAVA_HOME=$(/usr/libexec/java_home -v13)
+mvn deploy
diff --git a/bisect.sh b/bisect.sh
new file mode 100755
index 000000000..2afb2d7be
--- /dev/null
+++ b/bisect.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copyright (C) 2020-2022 Confluent, Inc.
+#
+
+set -x
+
+# It may be useful to make a copy of this file to run the bisect against a script outside the repository
+
+# tweak the working tree by merging the hot-fix branch
+# and then attempt a build
+# alterantively use cherry pick
+if git merge --no-commit --no-ff hot-fix &&
+ make; then
+ # run project specific test and report its status
+ ~/check_test_case.sh
+ status=$?
+else
+ # tell the caller this is untestable
+ status=125
+fi
+
+# undo the tweak to allow clean flipping to the next commit
+git reset --hard
+
+# return control
+exit $status
+
+mvn testCompile || exit 125 # this skips broken builds
+
+# run a maven test
+mvn -Dit.test=TransactionAndCommitModeTest#testLowMaxPoll -DskipUTs=true -DfailIfNoTests=false --projects parallel-consumer-core integration-test
diff --git a/checkcompile.sh b/checkcompile.sh
new file mode 100755
index 000000000..b94525242
--- /dev/null
+++ b/checkcompile.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright (C) 2020-2022 Confluent, Inc.
+#
+
+
+COMMITS=$(git log --oneline HEAD...182d13c43dec581a84c7edad962dfbd456744a64^ | cut -d " " -f 1)
+
+
+testCommit() {
+ # COMMIT = $1
+ echo Checking out commit $COMMIT
+ git checkout $COMMIT > /dev/null 2> /dev/null
+
+ mvn compile test-compile > /dev/null 2> /dev/null
+
+ if [ $? -eq 0 ]
+ then
+ echo $COMMIT passed
+ else
+ echo $COMMIT failed
+ fi
+}
+
+for COMMIT in $COMMITS
+do
+ testCommit "$COMMIT"
+done
+
+
diff --git a/mvnw b/mvnw
new file mode 100755
index 000000000..8d937f4c1
--- /dev/null
+++ b/mvnw
@@ -0,0 +1,308 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Apache Maven Wrapper startup batch script, version 3.2.0
+#
+# Required ENV vars:
+# ------------------
+# JAVA_HOME - location of a JDK home dir
+#
+# Optional ENV vars
+# -----------------
+# MAVEN_OPTS - parameters passed to the Java VM when running Maven
+# e.g. to debug Maven itself, use
+# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+# ----------------------------------------------------------------------------
+
+if [ -z "$MAVEN_SKIP_RC" ] ; then
+
+ if [ -f /usr/local/etc/mavenrc ] ; then
+ . /usr/local/etc/mavenrc
+ fi
+
+ if [ -f /etc/mavenrc ] ; then
+ . /etc/mavenrc
+ fi
+
+ if [ -f "$HOME/.mavenrc" ] ; then
+ . "$HOME/.mavenrc"
+ fi
+
+fi
+
+# OS specific support. $var _must_ be set to either true or false.
+cygwin=false;
+darwin=false;
+mingw=false
+case "$(uname)" in
+ CYGWIN*) cygwin=true ;;
+ MINGW*) mingw=true;;
+ Darwin*) darwin=true
+ # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
+ # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
+ if [ -z "$JAVA_HOME" ]; then
+ if [ -x "/usr/libexec/java_home" ]; then
+ JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME
+ else
+ JAVA_HOME="/Library/Java/Home"; export JAVA_HOME
+ fi
+ fi
+ ;;
+esac
+
+if [ -z "$JAVA_HOME" ] ; then
+ if [ -r /etc/gentoo-release ] ; then
+ JAVA_HOME=$(java-config --jre-home)
+ fi
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin ; then
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=$(cygpath --path --unix "$CLASSPATH")
+fi
+
+# For Mingw, ensure paths are in UNIX format before anything is touched
+if $mingw ; then
+ [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] &&
+ JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)"
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+ javaExecutable="$(which javac)"
+ if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then
+ # readlink(1) is not available as standard on Solaris 10.
+ readLink=$(which readlink)
+ if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then
+ if $darwin ; then
+ javaHome="$(dirname "\"$javaExecutable\"")"
+ javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac"
+ else
+ javaExecutable="$(readlink -f "\"$javaExecutable\"")"
+ fi
+ javaHome="$(dirname "\"$javaExecutable\"")"
+ javaHome=$(expr "$javaHome" : '\(.*\)/bin')
+ JAVA_HOME="$javaHome"
+ export JAVA_HOME
+ fi
+ fi
+fi
+
+if [ -z "$JAVACMD" ] ; then
+ if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ else
+ JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)"
+ fi
+fi
+
+if [ ! -x "$JAVACMD" ] ; then
+ echo "Error: JAVA_HOME is not defined correctly." >&2
+ echo " We cannot execute $JAVACMD" >&2
+ exit 1
+fi
+
+if [ -z "$JAVA_HOME" ] ; then
+ echo "Warning: JAVA_HOME environment variable is not set."
+fi
+
+# traverses directory structure from process work directory to filesystem root
+# first directory with .mvn subdirectory is considered project base directory
+find_maven_basedir() {
+ if [ -z "$1" ]
+ then
+ echo "Path not specified to find_maven_basedir"
+ return 1
+ fi
+
+ basedir="$1"
+ wdir="$1"
+ while [ "$wdir" != '/' ] ; do
+ if [ -d "$wdir"/.mvn ] ; then
+ basedir=$wdir
+ break
+ fi
+ # workaround for JBEAP-8937 (on Solaris 10/Sparc)
+ if [ -d "${wdir}" ]; then
+ wdir=$(cd "$wdir/.." || exit 1; pwd)
+ fi
+ # end of workaround
+ done
+ printf '%s' "$(cd "$basedir" || exit 1; pwd)"
+}
+
+# concatenates all lines of a file
+concat_lines() {
+ if [ -f "$1" ]; then
+ # Remove \r in case we run on Windows within Git Bash
+ # and check out the repository with auto CRLF management
+ # enabled. Otherwise, we may read lines that are delimited with
+ # \r\n and produce $'-Xarg\r' rather than -Xarg due to word
+ # splitting rules.
+ tr -s '\r\n' ' ' < "$1"
+ fi
+}
+
+log() {
+ if [ "$MVNW_VERBOSE" = true ]; then
+ printf '%s\n' "$1"
+ fi
+}
+
+BASE_DIR=$(find_maven_basedir "$(dirname "$0")")
+if [ -z "$BASE_DIR" ]; then
+ exit 1;
+fi
+
+MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR
+log "$MAVEN_PROJECTBASEDIR"
+
+##########################################################################################
+# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+# This allows using the maven wrapper in projects that prohibit checking in binary data.
+##########################################################################################
+wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar"
+if [ -r "$wrapperJarPath" ]; then
+ log "Found $wrapperJarPath"
+else
+ log "Couldn't find $wrapperJarPath, downloading it ..."
+
+ if [ -n "$MVNW_REPOURL" ]; then
+ wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
+ else
+ wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
+ fi
+ while IFS="=" read -r key value; do
+ # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' )
+ safeValue=$(echo "$value" | tr -d '\r')
+ case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;;
+ esac
+ done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties"
+ log "Downloading from: $wrapperUrl"
+
+ if $cygwin; then
+ wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath")
+ fi
+
+ if command -v wget > /dev/null; then
+ log "Found wget ... using wget"
+ [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet"
+ if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+ wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
+ else
+ wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
+ fi
+ elif command -v curl > /dev/null; then
+ log "Found curl ... using curl"
+ [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent"
+ if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+ curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath"
+ else
+ curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath"
+ fi
+ else
+ log "Falling back to using Java to download"
+ javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java"
+ javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class"
+ # For Cygwin, switch paths to Windows format before running javac
+ if $cygwin; then
+ javaSource=$(cygpath --path --windows "$javaSource")
+ javaClass=$(cygpath --path --windows "$javaClass")
+ fi
+ if [ -e "$javaSource" ]; then
+ if [ ! -e "$javaClass" ]; then
+ log " - Compiling MavenWrapperDownloader.java ..."
+ ("$JAVA_HOME/bin/javac" "$javaSource")
+ fi
+ if [ -e "$javaClass" ]; then
+ log " - Running MavenWrapperDownloader.java ..."
+ ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath"
+ fi
+ fi
+ fi
+fi
+##########################################################################################
+# End of extension
+##########################################################################################
+
+# If specified, validate the SHA-256 sum of the Maven wrapper jar file
+wrapperSha256Sum=""
+while IFS="=" read -r key value; do
+ case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;;
+ esac
+done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties"
+if [ -n "$wrapperSha256Sum" ]; then
+ wrapperSha256Result=false
+ if command -v sha256sum > /dev/null; then
+ if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then
+ wrapperSha256Result=true
+ fi
+ elif command -v shasum > /dev/null; then
+ if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then
+ wrapperSha256Result=true
+ fi
+ else
+ echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available."
+ echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties."
+ exit 1
+ fi
+ if [ $wrapperSha256Result = false ]; then
+ echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2
+ echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2
+ echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2
+ exit 1
+ fi
+fi
+
+MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME")
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=$(cygpath --path --windows "$CLASSPATH")
+ [ -n "$MAVEN_PROJECTBASEDIR" ] &&
+ MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR")
+fi
+
+# Provide a "standardized" way to retrieve the CLI args that will
+# work with both Windows and non-Windows executions.
+MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*"
+export MAVEN_CMD_LINE_ARGS
+
+WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+# shellcheck disable=SC2086 # safe args
+exec "$JAVACMD" \
+ $MAVEN_OPTS \
+ $MAVEN_DEBUG_OPTS \
+ -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
+ "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
+ ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
diff --git a/mvnw.cmd b/mvnw.cmd
new file mode 100644
index 000000000..f80fbad3e
--- /dev/null
+++ b/mvnw.cmd
@@ -0,0 +1,205 @@
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Apache Maven Wrapper startup batch script, version 3.2.0
+@REM
+@REM Required ENV vars:
+@REM JAVA_HOME - location of a JDK home dir
+@REM
+@REM Optional ENV vars
+@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
+@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
+@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
+@REM e.g. to debug Maven itself, use
+@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+@REM ----------------------------------------------------------------------------
+
+@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
+@echo off
+@REM set title of command window
+title %0
+@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
+@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
+
+@REM set %HOME% to equivalent of $HOME
+if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
+
+@REM Execute a user defined script before this one
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
+@REM check for pre script, once with legacy .bat ending and once with .cmd ending
+if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %*
+if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %*
+:skipRcPre
+
+@setlocal
+
+set ERROR_CODE=0
+
+@REM To isolate internal variables from possible post scripts, we use another setlocal
+@setlocal
+
+@REM ==== START VALIDATION ====
+if not "%JAVA_HOME%" == "" goto OkJHome
+
+echo.
+echo Error: JAVA_HOME not found in your environment. >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+:OkJHome
+if exist "%JAVA_HOME%\bin\java.exe" goto init
+
+echo.
+echo Error: JAVA_HOME is set to an invalid directory. >&2
+echo JAVA_HOME = "%JAVA_HOME%" >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+@REM ==== END VALIDATION ====
+
+:init
+
+@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
+@REM Fallback to current working directory if not found.
+
+set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
+IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
+
+set EXEC_DIR=%CD%
+set WDIR=%EXEC_DIR%
+:findBaseDir
+IF EXIST "%WDIR%"\.mvn goto baseDirFound
+cd ..
+IF "%WDIR%"=="%CD%" goto baseDirNotFound
+set WDIR=%CD%
+goto findBaseDir
+
+:baseDirFound
+set MAVEN_PROJECTBASEDIR=%WDIR%
+cd "%EXEC_DIR%"
+goto endDetectBaseDir
+
+:baseDirNotFound
+set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
+cd "%EXEC_DIR%"
+
+:endDetectBaseDir
+
+IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
+
+@setlocal EnableExtensions EnableDelayedExpansion
+for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
+@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
+
+:endReadAdditionalConfig
+
+SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
+set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
+set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
+
+FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
+ IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B
+)
+
+@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
+if exist %WRAPPER_JAR% (
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Found %WRAPPER_JAR%
+ )
+) else (
+ if not "%MVNW_REPOURL%" == "" (
+ SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
+ )
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Couldn't find %WRAPPER_JAR%, downloading it ...
+ echo Downloading from: %WRAPPER_URL%
+ )
+
+ powershell -Command "&{"^
+ "$webclient = new-object System.Net.WebClient;"^
+ "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
+ "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
+ "}"^
+ "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^
+ "}"
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Finished downloading %WRAPPER_JAR%
+ )
+)
+@REM End of extension
+
+@REM If specified, validate the SHA-256 sum of the Maven wrapper jar file
+SET WRAPPER_SHA_256_SUM=""
+FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
+ IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B
+)
+IF NOT %WRAPPER_SHA_256_SUM%=="" (
+ powershell -Command "&{"^
+ "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^
+ "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^
+ " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^
+ " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^
+ " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^
+ " exit 1;"^
+ "}"^
+ "}"
+ if ERRORLEVEL 1 goto error
+)
+
+@REM Provide a "standardized" way to retrieve the CLI args that will
+@REM work with both Windows and non-Windows executions.
+set MAVEN_CMD_LINE_ARGS=%*
+
+%MAVEN_JAVA_EXE% ^
+ %JVM_CONFIG_MAVEN_PROPS% ^
+ %MAVEN_OPTS% ^
+ %MAVEN_DEBUG_OPTS% ^
+ -classpath %WRAPPER_JAR% ^
+ "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^
+ %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
+if ERRORLEVEL 1 goto error
+goto end
+
+:error
+set ERROR_CODE=1
+
+:end
+@endlocal & set ERROR_CODE=%ERROR_CODE%
+
+if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost
+@REM check for post script, once with legacy .bat ending and once with .cmd ending
+if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat"
+if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd"
+:skipRcPost
+
+@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
+if "%MAVEN_BATCH_PAUSE%"=="on" pause
+
+if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE%
+
+cmd /C exit /B %ERROR_CODE%
diff --git a/parallel-consumer-core/pom.xml b/parallel-consumer-core/pom.xml
new file mode 100644
index 000000000..f3c39f4ea
--- /dev/null
+++ b/parallel-consumer-core/pom.xml
@@ -0,0 +1,168 @@
+
+
+
+
+ io.confluent.parallelconsumer
+ parallel-consumer-parent
+ 0.5.2.8-SNAPSHOT
+
+
+ 4.0.0
+
+ parallel-consumer-core
+ Confluent Parallel Consumer Core
+
+
+
+
+ org.apache.kafka
+ kafka-clients
+ ${kafka.version}
+
+
+ com.github.luben
+ zstd-jni
+ 1.5.5-4
+ compile
+
+
+ org.xerial.snappy
+ snappy-java
+ 1.1.10.3
+ compile
+
+
+ io.micrometer
+ micrometer-core
+ compile
+
+
+
+
+ org.awaitility
+ awaitility
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-params
+ test
+
+
+ org.junit-pioneer
+ junit-pioneer
+ test
+
+
+ org.testcontainers
+ testcontainers
+ test
+
+
+ org.testcontainers
+ kafka
+ test
+
+
+ org.testcontainers
+ junit-jupiter
+ test
+
+
+ org.apache.commons
+ commons-lang3
+ test
+
+
+ me.tongfei
+ progressbar
+ test
+
+
+ org.testcontainers
+ postgresql
+ test
+
+
+ org.postgresql
+ postgresql
+ 42.6.0
+ test
+
+
+ com.github.tomakehurst
+ wiremock-jre8
+ test
+
+
+ org.threeten
+ threeten-extra
+ 1.7.2
+ test
+
+
+ io.stubbs.truth
+ truth-generator-api
+ ${truth-generator-maven-plugin.version}
+ test
+
+
+ uk.co.jemos.podam
+ podam
+ 7.2.11.RELEASE
+ test
+
+
+
+
+
+
+ io.stubbs.truth
+ truth-generator-maven-plugin
+ ${truth-generator-maven-plugin.version}
+
+ true
+ false
+ true
+ 8
+
+ io.confluent.parallelconsumer.PollContext
+ io.confluent.parallelconsumer.ParallelEoSStreamProcessor
+ io.confluent.parallelconsumer.internal.ProducerManager
+ io.confluent.parallelconsumer.state.WorkContainer
+ io.confluent.parallelconsumer.state.WorkManager
+ io.confluent.parallelconsumer.state.PartitionState
+ io.confluent.parallelconsumer.state.ProcessingShard
+ io.confluent.parallelconsumer.state.ShardKey
+ io.confluent.parallelconsumer.offsets.OffsetEncoding
+
+
+
+ org.apache.kafka.clients.consumer.OffsetAndMetadata
+ org.apache.kafka.clients.consumer.ConsumerRecord
+ org.apache.kafka.clients.consumer.ConsumerRecords
+ org.apache.kafka.clients.consumer.Consumer
+ org.apache.kafka.clients.producer.RecordMetadata
+ org.apache.kafka.clients.producer.ProducerRecord
+ org.apache.kafka.clients.producer.Producer
+
+ io.confluent.parallelconsumer
+
+
+
+ generate-test-sources
+
+ generate
+
+
+
+
+
+
+
+
+
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/BackportUtils.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/BackportUtils.java
new file mode 100644
index 000000000..16c2c5e14
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/BackportUtils.java
@@ -0,0 +1,80 @@
+package io.confluent.csid.utils;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import lombok.experimental.UtilityClass;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Optional;
+
+@UtilityClass
+public class BackportUtils {
+
+ /**
+ * @see Duration#toSeconds() intro'd in Java 9 (isn't in 8)
+ */
+ public static long toSeconds(Duration duration) {
+ return duration.toMillis() / 1000;
+ }
+
+ /**
+ * @see Optional#isEmpty() intro'd java 11
+ */
+ public static boolean isEmpty(Optional> optional) {
+ return !optional.isPresent();
+ }
+
+
+ /**
+ * @see Optional#isEmpty() intro'd java 11
+ */
+ public static boolean hasNo(Optional> optional) {
+ return !optional.isPresent();
+ }
+
+ public static byte[] readFully(InputStream is) throws IOException {
+ return BackportUtils.readFully(is, -1, true);
+ }
+
+ /**
+ * Used in Java 8 environments (Java 9 has read all bytes)
+ *
+ * https://stackoverflow.com/a/25892791/105741
+ */
+ public static byte[] readFully(InputStream is, int length, boolean readAll) throws IOException {
+ byte[] output = {};
+ if (length == -1) length = Integer.MAX_VALUE;
+ int pos = 0;
+ while (pos < length) {
+ int bytesToRead;
+ if (pos >= output.length) { // Only expand when there's no room
+ bytesToRead = Math.min(length - pos, output.length + 1024);
+ if (output.length < pos + bytesToRead) {
+ output = Arrays.copyOf(output, pos + bytesToRead);
+ }
+ } else {
+ bytesToRead = output.length - pos;
+ }
+ int cc = is.read(output, pos, bytesToRead);
+ if (cc < 0) {
+ if (readAll && length != Integer.MAX_VALUE) {
+ throw new EOFException("Detect premature EOF");
+ } else {
+ if (output.length != pos) {
+ output = Arrays.copyOf(output, pos);
+ }
+ break;
+ }
+ }
+ pos += cc;
+ }
+ return output;
+ }
+
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/Java8StreamUtils.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/Java8StreamUtils.java
new file mode 100644
index 000000000..bf3f78d34
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/Java8StreamUtils.java
@@ -0,0 +1,45 @@
+package io.confluent.csid.utils;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import lombok.experimental.UtilityClass;
+
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+@UtilityClass
+public class Java8StreamUtils {
+
+ public static Stream setupStreamFromDeque(Deque extends T> userProcessResultsStream) {
+ Spliterator spliterator = Spliterators.spliterator(new DequeIterator(userProcessResultsStream), userProcessResultsStream.size(), Spliterator.NONNULL);
+
+ return StreamSupport.stream(spliterator, false);
+ }
+
+ private static class DequeIterator implements Iterator {
+
+ private final Deque extends T> userProcessResultsStream;
+
+ public DequeIterator(Deque extends T> userProcessResultsStream) {
+ this.userProcessResultsStream = userProcessResultsStream;
+ }
+
+ @Override
+ public boolean hasNext() {
+ boolean notEmpty = !userProcessResultsStream.isEmpty();
+ return notEmpty;
+ }
+
+ @Override
+ public T next() {
+ T poll = userProcessResultsStream.poll();
+ return poll;
+ }
+ }
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/JavaUtils.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/JavaUtils.java
new file mode 100644
index 000000000..7d1739e60
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/JavaUtils.java
@@ -0,0 +1,82 @@
+package io.confluent.csid.utils;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import io.confluent.parallelconsumer.internal.InternalRuntimeException;
+import lombok.experimental.UtilityClass;
+
+import java.time.Duration;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.time.Duration.ofMillis;
+
+@UtilityClass
+public class JavaUtils {
+
+ public static Optional getLast(final List someList) {
+ if (someList.isEmpty()) return Optional.empty();
+ return Optional.of(someList.get(someList.size() - 1));
+ }
+
+ public static Optional getFirst(final List someList) {
+ return someList.isEmpty() ? Optional.empty() : Optional.of(someList.get(0));
+ }
+
+ public static Optional getOnlyOne(final Map stringMapMap) {
+ if (stringMapMap.isEmpty()) return Optional.empty();
+ Collection values = stringMapMap.values();
+ if (values.size() > 1) throw new InternalRuntimeException("More than one element");
+ return Optional.of(values.iterator().next());
+ }
+
+ public static Duration max(Duration left, Duration right) {
+ long expectedDurationOfClose = Math.max(left.toMillis(), right.toMillis());
+ return ofMillis(expectedDurationOfClose);
+ }
+
+ public static boolean isGreaterThan(Duration compare, Duration to) {
+ return compare.compareTo(to) > 0;
+ }
+
+ /**
+ * A shortcut for changing only the values of a Map.
+ *
+ * https://stackoverflow.com/a/50740570/105741
+ */
+ public static Map remap(Map map,
+ Function super V1, ? extends V2> function) {
+ return map.entrySet()
+ .stream() // or parallel
+ .collect(Collectors.toMap(
+ Map.Entry::getKey,
+ e -> function.apply(e.getValue())
+ ));
+ }
+
+ public static List getRandom(List list, int quantity) {
+ if (list.size() < quantity) {
+ throw new IllegalArgumentException("List size is less than quantity");
+ }
+
+ return createRandomIntStream(list.size())
+ .limit(quantity)
+ .map(list::get)
+ .collect(Collectors.toList());
+ }
+
+ private static Stream createRandomIntStream(int range) {
+ final Random random = new Random();
+ return Stream.generate(() -> random.nextInt(range));
+ }
+
+ public static Collector> toTreeSet() {
+ return Collectors.toCollection(TreeSet::new);
+ }
+
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/KafkaUtils.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/KafkaUtils.java
new file mode 100644
index 000000000..2aa58614f
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/KafkaUtils.java
@@ -0,0 +1,21 @@
+package io.confluent.csid.utils;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import lombok.experimental.UtilityClass;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.TopicPartition;
+
+/**
+ * Simple identifier tuple for Topic Partitions
+ */
+@UtilityClass
+public final class KafkaUtils {
+
+ public static TopicPartition toTopicPartition(ConsumerRecord, ?> rec) {
+ return new TopicPartition(rec.topic(), rec.partition());
+ }
+
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/LoopingResumingIterator.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/LoopingResumingIterator.java
new file mode 100644
index 000000000..5258d70cd
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/LoopingResumingIterator.java
@@ -0,0 +1,173 @@
+package io.confluent.csid.utils;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Loop implementations that will resume from a given key. Can be constructed and used as an iterable, or a function
+ * passed into the static version {@link #iterateStartingFromKeyLooping}.
+ *
+ * Uses a looser contract than {@link Iterator} - that being it has no #hasNext() method - instead, it's {@link #next()}
+ * returns {@link Optional#empty()} when it's done.
+ *
+ * The non-functional version is useful when you want to use looping constructs such as {@code break} and
+ * {@code continue}.
+ *
+ *
+ * @author Antony Stubbs
+ */
+@Slf4j
+public class LoopingResumingIterator {
+
+ private Optional> head = Optional.empty();
+
+ /**
+ * See {@link java.util.concurrent.ConcurrentHashMap} docs on iteration
+ *
+ * @see java.util.concurrent.ConcurrentHashMap.Traverser
+ */
+ private Iterator> wrappedIterator;
+
+ /**
+ * As {@link java.util.concurrent.ConcurrentHashMap}'s iterators are thread safe, they see a snapshot of the map in
+ * time - this may cause the starting point key to be removed. In which case, we limit our iteration to taking the
+ * expected number of elements.
+ *
+ */
+ private final long iterationTargetCount;
+
+ /**
+ * The number of iterations we've done so far.
+ *
+ * @see #iterationTargetCount
+ */
+ private long iterationCount = 0;
+
+ /**
+ * The key to start from
+ */
+ @Getter
+ private final Optional iterationStartingPointKey;
+
+ private final Map map;
+
+ /**
+ * Where the iteration of the collection has now started again from index zero.
+ *
+ * Binary, as can only loop once after reach the end (to reach the initial starting point again).
+ */
+ private boolean isOnSecondPass = false;
+
+ /**
+ * Iteration has fully completed, and the collection is now exhausted.
+ */
+ private boolean terminalState = false;
+
+ /**
+ * A start key was provided, and it was found in the collection.
+ */
+ private boolean startingPointKeyValid = false;
+
+ public static LoopingResumingIterator build(KKEY startingKey, Map map) {
+ return new LoopingResumingIterator<>(Optional.ofNullable(startingKey), map);
+ }
+
+ /**
+ * Will resume from the startingKey, if it's present
+ */
+ @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
+ public LoopingResumingIterator(Optional startingKey, Map map) {
+ this.iterationStartingPointKey = startingKey;
+ this.map = map;
+ this.wrappedIterator = map.entrySet().iterator();
+ this.iterationTargetCount = map.size();
+
+ // find the starting point
+ if (startingKey.isPresent()) {
+ this.head = advanceToStartingPointAndGet(startingKey.get());
+ if (head.isPresent()) {
+ this.startingPointKeyValid = true;
+ } else {
+ resetIteratorToZero();
+ }
+ }
+ }
+
+ public LoopingResumingIterator(Map map) {
+ this(Optional.empty(), map);
+ }
+
+
+ /**
+ * @return null if no more elements
+ */
+ public Optional> next() {
+ iterationCount++;
+
+ // special cases
+ if (terminalState) {
+ return Optional.empty();
+ } else if (this.head.isPresent()) {
+ Optional> headSave = takeHeadValue();
+ return headSave;
+ }
+
+ if (wrappedIterator.hasNext()) {
+ Map.Entry next = wrappedIterator.next();
+ // could find the starting point earlier
+ boolean onSecondPassAndReachedStartingPoint = iterationStartingPointKey.equals(Optional.of(next.getKey()));
+ // or it could be missing entirely
+ boolean numberElementsReturnedExceeded = iterationCount > iterationTargetCount + 1; // off by one due to eager increment
+ if (onSecondPassAndReachedStartingPoint || numberElementsReturnedExceeded) {
+ // end second iteration reached
+ terminalState = true;
+ return Optional.empty();
+ } else {
+ return Optional.ofNullable(next);
+ }
+ } else if (iterationStartingPointKey.isPresent() && startingPointKeyValid && !isOnSecondPass) {
+ // we've reached the end, but we have a starting point set, so loop back to the start and do second pass
+ resetIteratorToZero();
+ isOnSecondPass = true;
+ return next();
+ } else {
+ // end of 2nd pass
+ return Optional.empty();
+ }
+ }
+
+ private Optional> takeHeadValue() {
+ var headSave = head;
+ head = Optional.empty();
+ return headSave;
+ }
+
+ /**
+ * Finds the starting point entry, and sets its index if found.
+ *
+ * @return the starting point entry, if found. Otherwise, null.
+ * @see #startingPointIndex
+ */
+ private Optional> advanceToStartingPointAndGet(Object startingPointObject) {
+ while (wrappedIterator.hasNext()) {
+ Map.Entry next = wrappedIterator.next();
+ if (next.getKey() == startingPointObject) {
+ return Optional.of(next);
+ }
+ }
+ return Optional.empty();
+ }
+
+ private void resetIteratorToZero() {
+ wrappedIterator = map.entrySet().iterator();
+ }
+
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/MathUtils.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/MathUtils.java
new file mode 100644
index 000000000..ecb7420ae
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/MathUtils.java
@@ -0,0 +1,29 @@
+package io.confluent.csid.utils;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import lombok.experimental.UtilityClass;
+
+/**
+ * @author Antony Stubbs
+ */
+@UtilityClass
+public class MathUtils {
+
+ /**
+ * Ensures exact conversion from a Long to a Short.
+ *
+ * {@link Math} doesn't have an exact conversion from Long to Short.
+ *
+ * @see Math#toIntExact
+ */
+ public static short toShortExact(long value) {
+ final short shortCast = (short) value;
+ if (shortCast != value) {
+ throw new ArithmeticException("short overflow");
+ }
+ return shortCast;
+ }
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/Range.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/Range.java
new file mode 100644
index 000000000..4edc0bab6
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/Range.java
@@ -0,0 +1,109 @@
+package io.confluent.csid.utils;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
+
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Class for simple ranges.
+ *
+ * For loop - like Python range function
+ *
+ * @see #range(long)
+ */
+public class Range implements Iterable {
+
+ private final long start;
+
+ private final long limit;
+
+ /**
+ * @see this#range(long)
+ */
+ public Range(int start, long max) {
+ this.start = start;
+ this.limit = max;
+ }
+
+ public Range(long limit) {
+ this.start = 0L;
+ this.limit = limit;
+ }
+
+ /**
+ * Provides an {@link Iterable} for the range of numbers from 0 to the given limit.
+ *
+ * Exclusive of max.
+ *
+ * Consider using {@link IntStream#range(int, int)#forEachOrdered} instead:
+ *
+ * However, if you don't want o use a closure, this is a good alternative.
+ */
+ public static Range range(long max) {
+ return new Range(max);
+ }
+
+ /**
+ * @see #range(long)
+ */
+ public static Range range(int start, long max) {
+ return new Range(start, max);
+ }
+
+ /**
+ * Potentially slow, but useful for tests
+ */
+ public static List listOfIntegers(int max) {
+ return Range.range(max).listAsIntegers();
+ }
+
+
+ @Override
+ public Iterator iterator() {
+ final long max = limit;
+ return new Iterator<>() {
+
+ private long current = start;
+
+ @Override
+ public boolean hasNext() {
+ return current < max;
+ }
+
+ @Override
+ public Long next() {
+ if (hasNext()) {
+ return current++;
+ } else {
+ throw new NoSuchElementException("Range reached the end");
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("Can't remove values from a Range");
+ }
+ };
+ }
+
+ public List listAsIntegers() {
+ return IntStream.range(Math.toIntExact(start), Math.toIntExact(limit))
+ .boxed()
+ .collect(toList());
+ }
+
+ public LongStream toStream() {
+ return LongStream.range(start, limit);
+ }
+
+}
\ No newline at end of file
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/StringUtils.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/StringUtils.java
new file mode 100644
index 000000000..012e388e5
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/StringUtils.java
@@ -0,0 +1,26 @@
+package io.confluent.csid.utils;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import lombok.experimental.UtilityClass;
+import org.slf4j.helpers.FormattingTuple;
+import org.slf4j.helpers.MessageFormatter;
+
+@UtilityClass
+public class StringUtils {
+
+ /**
+ * @see MessageFormatter#arrayFormat(String, Object[])
+ * @see FormattingTuple#getMessage()
+ */
+ public static String msg(String s, Object... args) {
+ return MessageFormatter.arrayFormat(s, args).getMessage();
+ }
+
+ public static boolean isBlank(final String property) {
+ if (property == null) return true;
+ else return property.trim().isEmpty(); // isBlank @since 11
+ }
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/SupplierUtils.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/SupplierUtils.java
new file mode 100644
index 000000000..d08616959
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/SupplierUtils.java
@@ -0,0 +1,33 @@
+package io.confluent.csid.utils;
+
+/*-
+ * Copyright (C) 2020-2023 Confluent, Inc.
+ */
+
+import lombok.experimental.UtilityClass;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+@UtilityClass
+public class SupplierUtils {
+
+ public static Supplier memoize(Supplier delegate) {
+ Objects.requireNonNull(delegate);
+ AtomicReference value = new AtomicReference<>();
+ return () -> {
+ T val = value.get();
+ if (val == null) {
+ synchronized (value) {
+ val = value.get();
+ if (val == null) {
+ val = Objects.requireNonNull(delegate.get());
+ value.set(val);
+ }
+ }
+ }
+ return val;
+ };
+ }
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/TimeUtils.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/TimeUtils.java
new file mode 100644
index 000000000..e9b3bd161
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/TimeUtils.java
@@ -0,0 +1,54 @@
+package io.confluent.csid.utils;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import lombok.Builder;
+import lombok.SneakyThrows;
+import lombok.Value;
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.concurrent.Callable;
+
+@Slf4j
+@UtilityClass
+public class TimeUtils {
+
+ public Clock getClock() {
+ return Clock.systemUTC();
+ }
+
+ @SneakyThrows
+ public static RESULT time(final Callable func) {
+ return timeWithMeta(func).getResult();
+ }
+
+ @SneakyThrows
+ public static TimeResult timeWithMeta(final Callable extends RESULT> func) {
+ long start = System.currentTimeMillis();
+ TimeResult.TimeResultBuilder timer = TimeResult.builder().startMs(start);
+ RESULT call = func.call();
+ timer.result(call);
+ long end = System.currentTimeMillis();
+ long elapsed = end - start;
+ timer.endMs(end);
+ log.trace("Function took {}", Duration.ofMillis(elapsed));
+ return timer.build();
+ }
+
+ @Builder
+ @Value
+ public static class TimeResult {
+ long startMs;
+ long endMs;
+ RESULT result;
+
+ public Duration getElapsed() {
+ return Duration.ofMillis(endMs - startMs);
+ }
+ }
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ExceptionInUserFunctionException.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ExceptionInUserFunctionException.java
new file mode 100644
index 000000000..82192f9ca
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ExceptionInUserFunctionException.java
@@ -0,0 +1,14 @@
+package io.confluent.parallelconsumer;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import lombok.experimental.StandardException;
+
+/**
+ * This exception is only used when there is an exception thrown from code provided by the user.
+ */
+@StandardException
+public class ExceptionInUserFunctionException extends ParallelConsumerException {
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/JStreamParallelEoSStreamProcessor.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/JStreamParallelEoSStreamProcessor.java
new file mode 100644
index 000000000..fa8d90847
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/JStreamParallelEoSStreamProcessor.java
@@ -0,0 +1,41 @@
+package io.confluent.parallelconsumer;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import io.confluent.csid.utils.Java8StreamUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.producer.ProducerRecord;
+
+import java.util.List;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+@Slf4j
+public class JStreamParallelEoSStreamProcessor extends ParallelEoSStreamProcessor implements JStreamParallelStreamProcessor {
+
+ private final Stream> stream;
+
+ private final ConcurrentLinkedDeque> userProcessResultsStream;
+
+ public JStreamParallelEoSStreamProcessor(ParallelConsumerOptions parallelConsumerOptions) {
+ super(parallelConsumerOptions);
+
+ this.userProcessResultsStream = new ConcurrentLinkedDeque<>();
+
+ this.stream = Java8StreamUtils.setupStreamFromDeque(this.userProcessResultsStream);
+ }
+
+ @Override
+ public Stream> pollProduceAndStream(Function, List>> userFunction) {
+ super.pollAndProduceMany(userFunction, result -> {
+ log.trace("Wrapper callback applied, sending result to stream. Input: {}", result);
+ this.userProcessResultsStream.add(result);
+ });
+
+ return this.stream;
+ }
+
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/JStreamParallelStreamProcessor.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/JStreamParallelStreamProcessor.java
new file mode 100644
index 000000000..be55c7d80
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/JStreamParallelStreamProcessor.java
@@ -0,0 +1,30 @@
+package io.confluent.parallelconsumer;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import io.confluent.parallelconsumer.internal.AbstractParallelEoSStreamProcessor;
+import io.confluent.parallelconsumer.internal.DrainingCloseable;
+import org.apache.kafka.clients.producer.ProducerRecord;
+
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+public interface JStreamParallelStreamProcessor extends DrainingCloseable {
+
+ static JStreamParallelStreamProcessor createJStreamEosStreamProcessor(ParallelConsumerOptions options) {
+ return new JStreamParallelEoSStreamProcessor<>(options);
+ }
+
+ /**
+ * Like {@link AbstractParallelEoSStreamProcessor#pollAndProduceMany} but instead of callbacks, streams the results
+ * instead, after the produce result is ack'd by Kafka.
+ *
+ * @return a stream of results of applying the function to the polled records
+ */
+ Stream> pollProduceAndStream(
+ Function,
+ List>> userFunction);
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/PCRetriableException.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/PCRetriableException.java
new file mode 100644
index 000000000..7fe1a3ad7
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/PCRetriableException.java
@@ -0,0 +1,23 @@
+package io.confluent.parallelconsumer;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import lombok.experimental.StandardException;
+
+/**
+ * A user's processing function can throw this exception, which signals to PC that processing of the message has failed,
+ * and that it should be retired at a later time.
+ *
+ * The advantage of throwing this exception explicitly, is that PC will not log an ERROR. If any other type of exception
+ * is thrown by the user's function, that will be logged as an error (but will still be retried later).
+ *
+ * So in short, if this exception is thrown, nothing will be logged (except at DEBUG level), any other exception will be
+ * logged as an error.
+ *
+ * @author Antony Stubbs
+ */
+@StandardException
+public class PCRetriableException extends RuntimeException {
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumer.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumer.java
new file mode 100644
index 000000000..de95f1f98
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumer.java
@@ -0,0 +1,101 @@
+package io.confluent.parallelconsumer;
+
+/*-
+ * Copyright (C) 2020-2022 Confluent, Inc.
+ */
+
+import io.confluent.parallelconsumer.internal.AbstractParallelEoSStreamProcessor;
+import io.confluent.parallelconsumer.internal.DrainingCloseable;
+import lombok.Data;
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+
+import java.util.Collection;
+import java.util.regex.Pattern;
+
+// tag::javadoc[]
+/**
+ * Asynchronous / concurrent message consumer for Kafka.
+ *
+ * Currently, there is no direct implementation, only the {@link ParallelStreamProcessor} version (see
+ * {@link AbstractParallelEoSStreamProcessor}), but there may be in the future.
+ *
+ * @param key consume / produce key type
+ * @param value consume / produce value type
+ * @see AbstractParallelEoSStreamProcessor
+ */
+// end::javadoc[]
+public interface ParallelConsumer extends DrainingCloseable {
+
+ /**
+ * @return true if the system has either closed, or has crashed
+ */
+ boolean isClosedOrFailed();
+
+ /**
+ * @see KafkaConsumer#subscribe(Collection)
+ */
+ void subscribe(Collection topics);
+
+ /**
+ * @see KafkaConsumer#subscribe(Pattern)
+ */
+ void subscribe(Pattern pattern);
+
+ /**
+ * @see KafkaConsumer#subscribe(Collection, ConsumerRebalanceListener)
+ */
+ void subscribe(Collection topics, ConsumerRebalanceListener callback);
+
+ /**
+ * @see KafkaConsumer#subscribe(Pattern, ConsumerRebalanceListener)
+ */
+ void subscribe(Pattern pattern, ConsumerRebalanceListener callback);
+
+ /**
+ * Pause this consumer (i.e. stop processing of messages).
+ *
+ * This operation only has an effect if the consumer is currently running. In all other cases calling this method
+ * will be silent a no-op.
+ *
+ * Once the consumer is paused, the system will stop submitting work to the processing pool. Already submitted in
+ * flight work however will be finished. This includes work that is currently being processed inside a user function
+ * as well as work that has already been submitted to the processing pool but has not been picked up by a free
+ * worker yet.
+ *
+ * General remarks:
+ *
+ *
A paused consumer may still keep polling for new work until internal buffers are filled.
+ *
This operation does not actively pause the subscription on the underlying Kafka Broker (compared to
+ * {@link KafkaConsumer#pause KafkaConsumer#pause}).
+ *
Pending offset commits will still be performed when the consumer is paused.
+ *
+ */
+ void pauseIfRunning();
+
+ /**
+ * Resume this consumer (i.e. continue processing of messages).
+ *
+ * This operation only has an effect if the consumer is currently paused. In all other cases calling this method
+ * will be a silent no-op.
+ *
+ * If you want to go deeper, look at {@link #defaultMessageRetryDelay}, {@link #retryDelayProvider} and
+ * {@link #commitMode}.
+ *
+ * Note: The only required option is the {@link #consumer} ({@link #producer} is only needed if you use the Produce
+ * flows). All other options have sensible defaults.
+ *
+ * @author Antony Stubbs
+ * @see #builder()
+ * @see ParallelConsumerOptions.ParallelConsumerOptionsBuilder
+ */
+@Getter
+@Builder(toBuilder = true)
+@ToString
+@FieldNameConstants
+@InterfaceStability.Evolving
+public class ParallelConsumerOptions {
+
+ /**
+ * Required parameter for all use.
+ */
+ private final Consumer consumer;
+
+ /**
+ * Supplying a producer is only needed if using the produce flows.
+ *
+ * @see ParallelStreamProcessor
+ */
+ private final Producer producer;
+
+ /**
+ * Path to Managed executor service for Java EE
+ */
+ @Builder.Default
+ private final String managedExecutorService = "java:comp/DefaultManagedExecutorService";
+
+ /**
+ * Path to Managed thread factory for Java EE
+ */
+ @Builder.Default
+ private final String managedThreadFactory = "java:comp/DefaultManagedThreadFactory";
+
+ /**
+ * Micrometer MeterRegistry
+ *
+ * Optional - if not specified CompositeMeterRegistry will be used which is NoOp
+ */
+ private final MeterRegistry meterRegistry;
+
+ /**
+ * PC Instance metrics tag value - if specified - should be unique to allow instance specific meters to be created
+ * and cleared. Used with Tag key {@link PCMetricsDef#PC_INSTANCE_TAG}
+ *
+ * If not set - unique UUID will be generated for it
+ */
+ private final String pcInstanceTag;
+
+ /**
+ * Additional common metrics tags - will be added to all created meters
+ */
+ @Builder.Default
+ private final Iterable metricsTags = Tags.empty();
+
+ /**
+ * The ordering guarantee to use.
+ */
+ public enum ProcessingOrder {
+
+ /**
+ * No ordering is guaranteed, not even partition order. Fastest. Concurrency is at most the max number of
+ * concurrency or max number of uncommitted messages, limited by the max concurrency or uncommitted settings.
+ */
+ UNORDERED,
+
+ /**
+ * Process messages within a partition in order, but process multiple partitions in parallel. Similar to running
+ * more consumer for a topic. Concurrency is at most the number of partitions.
+ */
+ PARTITION,
+
+ /**
+ * Process messages in key order. Concurrency is at most the number of unique keys in a topic, limited by the
+ * max concurrency or uncommitted settings.
+ */
+ KEY
+ }
+
+ /**
+ * The type of commit to be made, with either a transactions configured Producer where messages produced are
+ * committed back to the Broker along with the offsets they originated from, or with the faster simpler Consumer
+ * offset system either synchronously or asynchronously
+ */
+ public enum CommitMode {
+
+ // tag::transactionalJavadoc[]
+ /**
+ * Periodically commits through the Producer using transactions.
+ *
+ * Messages sent in parallel by different workers get added to the same transaction block - you end up with
+ * transactions 100ms (by default) "large", containing all records sent during that time period, from the
+ * offsets being committed.
+ *
+ * Of no use, if not also producing messages (i.e. using a {@link ParallelStreamProcessor#pollAndProduce}
+ * variation).
+ *
+ * Note: Records being sent by different threads will all be in a single transaction, as PC shares a single
+ * Producer instance. This could be seen as a performance overhead advantage, efficient resource use, in
+ * exchange for a loss in transaction granularity.
+ *
+ * The benefits of using this mode are:
+ *
+ * a) All records produced from a given source offset will either all be visible, or none will be
+ * ({@link org.apache.kafka.common.IsolationLevel#READ_COMMITTED}).
+ *
+ * b) If any records making up a transaction have a terminal issue being produced, or the system crashes before
+ * finishing sending all the records and committing, none will ever be visible and the system will eventually
+ * retry them in new transactions - potentially with different combinations of records from the original.
+ *
+ * c) A source offset, and it's produced records will be committed as an atomic set. Normally: either the record
+ * producing could fail, or the committing of the source offset could fail, as they are separate individual
+ * operations. When using Transactions, they are committed together - so if either operations fails, the
+ * transaction will never get committed, and upon recovery, the system will retry the set again (and no
+ * duplicates will be visible in the topic).
+ *
+ * This {@code CommitMode} is the slowest of the options, but there will be no duplicates in Kafka caused by
+ * producing a record multiple times if previous offset commits have failed or crashes have occurred (however
+ * message replay may cause duplicates in external systems which is unavoidable - external systems must be
+ * idempotent).
+ *
+ * The default commit interval {@link AbstractParallelEoSStreamProcessor#KAFKA_DEFAULT_AUTO_COMMIT_FREQUENCY}
+ * gets automatically reduced from the default of 5 seconds to 100ms (the same as Kafka Streams commit.interval.ms).
+ * Reducing this configuration places higher load on the broker, but will reduce (but cannot eliminate) replay
+ * upon failure. Note also that when using transactions in Kafka, consumption in {@code READ_COMMITTED} mode is
+ * blocked up to the offset of the first STILL open transaction. Using a smaller commit frequency reduces this
+ * minimum consumption latency - the faster transactions are closed, the faster the transaction content can be
+ * read by {@code READ_COMMITTED} consumers. More information about this can be found on the Confluent blog
+ * post:
+ * Enabling Exactly-Once in Kafka
+ * Streams.
+ *
+ * When producing multiple records (see {@link ParallelStreamProcessor#pollAndProduceMany}), all records must
+ * have been produced successfully to the broker before the transaction will commit, after which all will be
+ * visible together, or none.
+ *
+ * Records produced while running in this mode, won't be seen by consumer running in
+ * {@link ConsumerConfig#ISOLATION_LEVEL_CONFIG} {@link org.apache.kafka.common.IsolationLevel#READ_COMMITTED}
+ * mode until the transaction is complete and all records are produced successfully. Records produced into a
+ * transaction that gets aborted or timed out, will never be visible.
+ *
+ * The system must prevent records from being produced to the brokers whose source consumer record offsets has
+ * not been included in this transaction. Otherwise, the transactions would include produced records from
+ * consumer offsets which would only be committed in the NEXT transaction, which would break the EoS guarantees.
+ * To achieve this, first work processing and record producing is suspended (by acquiring the commit lock -
+ * see{@link #commitLockAcquisitionTimeout}, as record processing requires the produce lock), then succeeded
+ * consumer offsets are gathered, transaction commit is made, then when the transaction has finished, processing
+ * resumes by releasing the commit lock. This periodically slows down record production during this phase, by
+ * the time needed to commit the transaction.
+ *
+ * This is all separate from using an IDEMPOTENT Producer, which can be used, along with the
+ * {@link ParallelConsumerOptions#commitMode} {@link CommitMode#PERIODIC_CONSUMER_SYNC} or
+ * {@link CommitMode#PERIODIC_CONSUMER_ASYNCHRONOUS}.
+ *
+ * Failure:
+ *
+ * Commit lock: If the system cannot acquire the commit lock in time, it will shut down for whatever reason, the
+ * system will shut down (fail fast) - during the shutdown a final commit attempt will be made. The default
+ * timeout for acquisition is very high though - see {@link #commitLockAcquisitionTimeout}. This can be caused
+ * by the user processing function taking too long to complete.
+ *
+ * Produce lock: If the system cannot acquire the produce lock in time, it will fail the record processing and
+ * retry the record later. This can be caused by the controller taking too long to commit for some reason. See
+ * {@link #produceLockAcquisitionTimeout}. If using {@link #allowEagerProcessingDuringTransactionCommit}, this
+ * may cause side effect replay when the record is retried, otherwise there is no replay. See
+ * {@link #allowEagerProcessingDuringTransactionCommit} for more details.
+ *
+ * @see ParallelConsumerOptions.ParallelConsumerOptionsBuilder#commitInterval
+ */
+ // end::transactionalJavadoc[]
+ PERIODIC_TRANSACTIONAL_PRODUCER,
+
+ /**
+ * Periodically synchronous commits with the Consumer. Much faster than
+ * {@link #PERIODIC_TRANSACTIONAL_PRODUCER}. Slower but potentially fewer duplicates than
+ * {@link #PERIODIC_CONSUMER_ASYNCHRONOUS} upon replay.
+ */
+ PERIODIC_CONSUMER_SYNC,
+
+ /**
+ * Periodically commits offsets asynchronously. The fastest option, under normal conditions will have few or no
+ * duplicates. Under failure recovery may have more duplicates than {@link #PERIODIC_CONSUMER_SYNC}.
+ */
+ PERIODIC_CONSUMER_ASYNCHRONOUS
+
+ }
+
+ /**
+ * Kafka's default auto commit interval - which is 5000ms.
+ *
+ * @see org.apache.kafka.clients.consumer.ConsumerConfig#AUTO_COMMIT_INTERVAL_MS_CONFIG
+ * @see org.apache.kafka.clients.consumer.ConsumerConfig#CONFIG
+ */
+ public static final int KAFKA_DEFAULT_AUTO_COMMIT_INTERVAL_MS = 5000;
+
+ public static final Duration DEFAULT_COMMIT_INTERVAL = ofMillis(KAFKA_DEFAULT_AUTO_COMMIT_INTERVAL_MS);
+
+ /*
+ * The same as Kafka Streams
+ */
+ public static final Duration DEFAULT_COMMIT_INTERVAL_FOR_TRANSACTIONS = ofMillis(100);
+
+ /**
+ * When using {@link CommitMode#PERIODIC_TRANSACTIONAL_PRODUCER}, allows new records to be processed UP UNTIL the
+ * result record SENDING ({@link Producer#send}) step, potentially while a transaction is being committed. Disabled
+ * by default as to prevent replay side effects when records need to be retried in some scenarios.
+ *
+ * Doesn't interfere with the transaction itself, just reduces side effects.
+ *
+ * Recommended to leave this off to avoid side effect duplicates upon rebalances after a crash. Enabling could
+ * improve performance as the produce lock will only be taken right before it's needed (optimistic locking) to
+ * produce the result record, instead of pessimistically locking.
+ */
+ @Builder.Default
+ private boolean allowEagerProcessingDuringTransactionCommit = false;
+
+ /**
+ * Time to allow for acquiring the commit lock. If record processing or producing takes a long time, you may need to
+ * increase this. If this fails, the system will shut down (fail fast) and attempt to commit once more.
+ */
+ @Builder.Default
+ private Duration commitLockAcquisitionTimeout = Duration.ofMinutes(5);
+
+ /**
+ * Time to allow for acquiring the produce lock. If transaction committing a long time, you may need to increase
+ * this. If this fails, the record will be returned to the processing queue for later retry.
+ */
+ @Builder.Default
+ private Duration produceLockAcquisitionTimeout = Duration.ofMinutes(1);
+
+ /**
+ * Time between commits. Using a higher frequency (a lower value) will put more load on the brokers.
+ */
+ @Builder.Default
+ private Duration commitInterval = DEFAULT_COMMIT_INTERVAL;
+
+ /**
+ * @deprecated only settable during {@code deprecation phase} - use
+ * {@link ParallelConsumerOptions.ParallelConsumerOptionsBuilder#commitInterval}} instead.
+ */
+ // todo delete in next major version
+ @Deprecated
+ public void setCommitInterval(Duration commitInterval) {
+ this.commitInterval = commitInterval;
+ }
+
+ /**
+ * The {@link ProcessingOrder} type to use
+ */
+ @Builder.Default
+ private final ProcessingOrder ordering = ProcessingOrder.KEY;
+
+ /**
+ * The {@link CommitMode} to be used
+ */
+ @Builder.Default
+ private final CommitMode commitMode = CommitMode.PERIODIC_CONSUMER_ASYNCHRONOUS;
+
+ /**
+ * Controls the maximum degree of concurrency to occur. Used to limit concurrent calls to external systems to a
+ * maximum to prevent overloading them or to a degree, using up quotas.
+ *
+ * When using {@link #getBatchSize()}, this is over and above the batch size setting. So for example, a
+ * {@link #getMaxConcurrency()} of {@code 2} and a batch size of {@code 3} would result in at most {@code 15}
+ * records being processed at once.
+ *
+ * A note on quotas - if your quota is expressed as maximum concurrent calls, this works well. If it's limited in
+ * total requests / sec, this may still overload the system. See towards the distributed rate limiting feature for
+ * this to be properly addressed: https://github.com/confluentinc/parallel-consumer/issues/24 Add distributed rate
+ * limiting support #24.
+ *
+ * In the core module, this sets the number of threads to use in the core's thread pool.
+ *
+ * It's recommended to set this quite high, much higher than core count, as it's expected that these threads will
+ * spend most of their time blocked waiting for IO. For automatic setting of this variable, look out for issue
+ * https://github.com/confluentinc/parallel-consumer/issues/21 Dynamic concurrency control with flow control or tcp
+ * congestion control theory #21.
+ */
+ @Builder.Default
+ private final int maxConcurrency = DEFAULT_MAX_CONCURRENCY;
+
+ public static final int DEFAULT_MAX_CONCURRENCY = 16;
+
+ public static final Duration DEFAULT_STATIC_RETRY_DELAY = Duration.ofSeconds(1);
+
+ /**
+ * Error handling strategy to use when invalid offsets metadata is encountered. This could happen accidentally or
+ * deliberately if the user attempts to reuse an existing consumer group id.
+ */
+ public enum InvalidOffsetMetadataHandlingPolicy {
+ /**
+ * Fails and shuts down the application. This is the default.
+ */
+ FAIL,
+ /**
+ * Ignore the error, logs a warning message and continue processing from the last committed offset.
+ */
+ IGNORE
+ }
+
+ /**
+ * Controls the error handling behaviour to use when invalid offsets metadata from a pre-existing consumer group is
+ * encountered. A potential scenario where this could occur is when a consumer group id from a Kafka Streams
+ * application is accidentally reused.
+ *
+ * Default is {@link InvalidOffsetMetadataHandlingPolicy#FAIL}
+ */
+ @Builder.Default
+ private final InvalidOffsetMetadataHandlingPolicy invalidOffsetMetadataPolicy = InvalidOffsetMetadataHandlingPolicy.FAIL;
+ /**
+ * When a message fails, how long the system should wait before trying that message again. Note that this will not
+ * be exact, and is just a target.
+ *
+ * @deprecated will be renamed to static retry delay
+ */
+ @Deprecated
+ @Builder.Default
+ private final Duration defaultMessageRetryDelay = DEFAULT_STATIC_RETRY_DELAY;
+
+ /**
+ * When present, use this to generate a dynamic retry delay, instead of a static one with
+ * {@link #getDefaultMessageRetryDelay()}.
+ *
+ * Overrides {@link #defaultMessageRetryDelay}, even if it's set.
+ */
+ private final Function, Duration> retryDelayProvider;
+
+ /**
+ * Controls how long to block while waiting for the {@link Producer#send} to complete for any ProducerRecords
+ * returned from the user-function. Only relevant if using one of the produce-flows and providing a
+ * {@link ParallelConsumerOptions#producer}. If the timeout occurs the record will be re-processed in the
+ * user-function.
+ *
+ * Consider aligning the value with the {@link ParallelConsumerOptions#producer}-options to avoid unnecessary
+ * re-processing and duplicates on slow {@link Producer#send} calls.
+ *
+ * @see org.apache.kafka.clients.producer.ProducerConfig#DELIVERY_TIMEOUT_MS_CONFIG
+ */
+ @Builder.Default
+ private final Duration sendTimeout = Duration.ofSeconds(10);
+
+ /**
+ * Controls how long to block while waiting for offsets to be committed. Only relevant if using
+ * {@link CommitMode#PERIODIC_CONSUMER_SYNC} commit-mode.
+ */
+ @Builder.Default
+ private final Duration offsetCommitTimeout = Duration.ofSeconds(10);
+
+ /**
+ * The maximum number of messages to attempt to pass into the user functions.
+ *
+ * Batch sizes may sometimes be less than this size, but will never be more.
+ *
+ * The system will treat the messages as a set, so if an error is thrown by the user code, then all messages will be
+ * marked as failed and be retried (Note that when they are retried, there is no guarantee they will all be in the
+ * same batch again). So if you're going to process messages individually, then don't set a batch size.
+ *
+ * Otherwise, if you're going to process messages in sub sets from this batch, it's better to instead adjust the
+ * {@link ParallelConsumerOptions#getBatchSize()} instead to the actual desired size, and process them as a whole.
+ *
+ * Note that there is no relationship between the {@link ConsumerConfig} setting of
+ * {@link ConsumerConfig#MAX_POLL_RECORDS_CONFIG} and this configured batch size, as this library introduces a large
+ * layer of indirection between the managed consumer, and the managed queues we use.
+ *
+ * This indirection effectively disconnects the processing of messages from "polling" them from the managed client,
+ * as we do not wait to process them before calling poll again. We simply call poll as much as we need to, in order
+ * to keep our queues full of enough work to satisfy demand.
+ *
+ * If we have enough, then we actively manage pausing our subscription so that we can continue calling {@code poll}
+ * without pulling in even more messages.
+ *
+ *
+ * @see ParallelConsumerOptions#getBatchSize()
+ */
+ @Builder.Default
+ private final Integer batchSize = 1;
+
+ /**
+ * Configure the amount of delay a record experiences, before a warning is logged.
+ */
+ @Builder.Default
+ private final Duration thresholdForTimeSpendInQueueWarning = Duration.ofSeconds(10);
+
+ public boolean isUsingBatching() {
+ return getBatchSize() > 1;
+ }
+
+ @Builder.Default
+ private final int maxFailureHistory = 10;
+
+ /**
+ * @return the combined target of the desired concurrency by the configured batch size
+ */
+ public int getTargetAmountOfRecordsInFlight() {
+ return getMaxConcurrency() * getBatchSize();
+ }
+
+ public void validate() {
+ Objects.requireNonNull(consumer, "A consumer must be supplied");
+
+ transactionsValidation();
+ }
+
+ private void transactionsValidation() {
+ boolean commitInternalHasNotBeenSet = getCommitInterval() == DEFAULT_COMMIT_INTERVAL;
+
+ if (isUsingTransactionCommitMode()) {
+ if (producer == null) {
+ throw new IllegalArgumentException(msg("Cannot set {} to Transaction Producer mode ({}) without supplying a Producer instance",
+ Fields.commitMode,
+ commitMode));
+ }
+
+ // update commit frequency
+ if (commitInternalHasNotBeenSet) {
+ this.commitInterval = DEFAULT_COMMIT_INTERVAL_FOR_TRANSACTIONS;
+ }
+ }
+
+ // inverse
+ if (!isUsingTransactionCommitMode()) {
+ if (isAllowEagerProcessingDuringTransactionCommit()) {
+ throw new IllegalArgumentException(msg("Cannot set {} (eager record processing) when not using transactional commit mode ({}={}).",
+ Fields.allowEagerProcessingDuringTransactionCommit,
+ Fields.commitMode,
+ commitMode));
+ }
+ }
+ }
+
+ /**
+ * @deprecated use {@link #isUsingTransactionCommitMode()}
+ */
+ @Deprecated
+ public boolean isUsingTransactionalProducer() {
+ return isUsingTransactionCommitMode();
+ }
+
+ /**
+ * @see CommitMode#PERIODIC_TRANSACTIONAL_PRODUCER
+ */
+ public boolean isUsingTransactionCommitMode() {
+ return commitMode.equals(PERIODIC_TRANSACTIONAL_PRODUCER);
+ }
+
+ public boolean isProducerSupplied() {
+ return getProducer() != null;
+ }
+
+ /**
+ * Timeout for shutting down execution pool during shutdown in DONT_DRAIN mode. Should be high enough to allow for
+ * inflight messages to finish processing, but low enough to kill any blocked thread to allow to rebalance in a
+ * timely manner, especially if shutting down on error.
+ */
+ @Builder.Default
+ public final Duration shutdownTimeout = Duration.ofSeconds(10);
+
+ /**
+ * Timeout for draining queue during shutdown in DRAIN mode. Should be high enough to allow for all queued messages
+ * to process.
+ */
+ @Builder.Default
+ public final Duration drainTimeout = Duration.ofSeconds(30);
+}
diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessor.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessor.java
new file mode 100644
index 000000000..8df85f171
--- /dev/null
+++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessor.java
@@ -0,0 +1,165 @@
+package io.confluent.parallelconsumer;
+
+/*-
+ * Copyright (C) 2020-2023 Confluent, Inc.
+ */
+
+import io.confluent.csid.utils.TimeUtils;
+import io.confluent.parallelconsumer.internal.AbstractParallelEoSStreamProcessor;
+import io.confluent.parallelconsumer.internal.InternalRuntimeException;
+import io.confluent.parallelconsumer.internal.PCModule;
+import io.confluent.parallelconsumer.internal.ProducerManager;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import pl.tlinkowski.unij.api.UniLists;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import static io.confluent.csid.utils.StringUtils.msg;
+import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.PERIODIC_TRANSACTIONAL_PRODUCER;
+import static io.confluent.parallelconsumer.internal.UserFunctions.carefullyRun;
+import static java.util.Optional.of;
+
+@Slf4j
+public class ParallelEoSStreamProcessor extends AbstractParallelEoSStreamProcessor
+ implements ParallelStreamProcessor {
+
+ /**
+ * Construct the AsyncConsumer by wrapping this passed in consumer and producer, which can be configured any which
+ * way as per normal.
+ *
+ * @see ParallelConsumerOptions
+ */
+ public ParallelEoSStreamProcessor(ParallelConsumerOptions newOptions, PCModule module) {
+ super(newOptions, module);
+ }
+
+ public ParallelEoSStreamProcessor(ParallelConsumerOptions newOptions) {
+ super(newOptions);
+ }
+
+ @Override
+ public void poll(Consumer> usersVoidConsumptionFunction) {
+ Function, List