From e143333bf122c6a3e3371c1d04af978f5f1049ac Mon Sep 17 00:00:00 2001 From: Nicolas Pepin-Perreault Date: Sat, 12 Jun 2021 14:58:53 +0200 Subject: [PATCH 1/2] test(qa): update to Zeebe 1.0.1 Updates QA tests to use Zeebe 1.0.1 and load the exporter as an external exporter and not via the class path. --- .../java/io/zeebe/exporters/kafka/qa/KafkaExporterIT.java | 6 ++++-- qa/src/test/resources/exporters.yml | 1 + qa/src/test/resources/simplelogger.properties | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/qa/src/test/java/io/zeebe/exporters/kafka/qa/KafkaExporterIT.java b/qa/src/test/java/io/zeebe/exporters/kafka/qa/KafkaExporterIT.java index b98e8eb..01afa32 100644 --- a/qa/src/test/java/io/zeebe/exporters/kafka/qa/KafkaExporterIT.java +++ b/qa/src/test/java/io/zeebe/exporters/kafka/qa/KafkaExporterIT.java @@ -20,6 +20,7 @@ import io.camunda.zeebe.client.ZeebeClient; import io.camunda.zeebe.protocol.record.Record; import io.zeebe.containers.ZeebeContainer; +import io.zeebe.containers.ZeebeDefaults; import io.zeebe.exporters.kafka.serde.RecordDeserializer; import io.zeebe.exporters.kafka.serde.RecordId; import io.zeebe.exporters.kafka.serde.RecordIdDeserializer; @@ -271,7 +272,8 @@ private DebugHttpExporterClient getLazyDebugExporter() throws MalformedURLExcept @SuppressWarnings("OctalInteger") private ZeebeContainer newZeebeContainer() { - final var container = new ZeebeContainer(); + final var container = + new ZeebeContainer(ZeebeDefaults.getInstance().getDefaultDockerImage().withTag("1.0.1")); final var exporterJar = MountableFile.forClasspathResource("zeebe-kafka-exporter.jar", 0775); final var exporterConfig = MountableFile.forClasspathResource("exporters.yml", 0775); final var loggingConfig = MountableFile.forClasspathResource("log4j2.xml", 0775); @@ -289,7 +291,7 @@ private ZeebeContainer newZeebeContainer() { .withEnv( "LOG4J_CONFIGURATION_FILE", "/usr/local/zeebe/config/log4j2.xml,/usr/local/zeebe/config/log4j2-exporter.xml") - .withCopyFileToContainer(exporterJar, "/usr/local/zeebe/lib/zeebe-kafka-exporter.jar") + .withCopyFileToContainer(exporterJar, "/usr/local/zeebe/exporters/zeebe-kafka-exporter.jar") .withCopyFileToContainer(exporterConfig, "/usr/local/zeebe/config/exporters.yml") .withCopyFileToContainer(loggingConfig, "/usr/local/zeebe/config/log4j2-exporter.xml") .withEnv("SPRING_CONFIG_ADDITIONAL_LOCATION", "file:/usr/local/zeebe/config/exporters.yml") diff --git a/qa/src/test/resources/exporters.yml b/qa/src/test/resources/exporters.yml index 4a32215..d8f2b70 100644 --- a/qa/src/test/resources/exporters.yml +++ b/qa/src/test/resources/exporters.yml @@ -7,6 +7,7 @@ zeebe: port: 8000 kafka: className: io.zeebe.exporters.kafka.KafkaExporter + jarPath: /usr/local/zeebe/exporters/zeebe-kafka-exporter.jar args: maxBatchSize: 100 maxBlockingTimeoutMs: 1000 diff --git a/qa/src/test/resources/simplelogger.properties b/qa/src/test/resources/simplelogger.properties index 6351ae9..37a9243 100644 --- a/qa/src/test/resources/simplelogger.properties +++ b/qa/src/test/resources/simplelogger.properties @@ -2,7 +2,7 @@ org.slf4j.simpleLogger.logFile=System.out org.slf4j.simplerLogger.showShortLogName=true org.slf4j.simpleLogger.defaultLogLevel=info org.slf4j.simpleLogger.log.io.zeebe.exporters.kafka.qa.KafkaExporterIT=debug -org.slf4j.simpleLogger.log.io.zeebe.exporters.kafka.qa.KafkaExporterIT.zeebeContainer=info +org.slf4j.simpleLogger.log.io.zeebe.exporters.kafka.qa.KafkaExporterIT.zeebeContainer=warn org.slf4j.simpleLogger.log.io.zeebe.exporters.kafka.qa.KafkaExporterIT.kafkaContainer=warn org.slf4j.simpleLogger.log.org.apache=warn org.slf4j.simpleLogger.log.kafka=warn From 279d1f854d8598f3a002a1d63d98d8956c356784 Mon Sep 17 00:00:00 2001 From: Nicolas Pepin-Perreault Date: Sat, 12 Jun 2021 14:59:30 +0200 Subject: [PATCH 2/2] docs: update note about external exporter JAR Updates docs to notify users that from 1.0.1, Zeebe can correctly load the Kafka exporter as an external JAR and it's not necessary to load directly from within the classpath. This helps avoid potential dependency version conflicts. --- README.md | 34 ++++++++++++++++++++++++++++------ exporter/exporter.yml | 3 +++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f718a1a..814b240 100644 --- a/README.md +++ b/README.md @@ -63,16 +63,35 @@ At the moment, the only guarantees for backwards compatibility are: The quickest way to get started is: 1. Download the latest release (`zeebe-kafka-exporter-*-jar-with-dependencies.jar`). -1. Copy it to the `lib/` folder of your Zeebe brokers. -1. Copy the contents of the configuration from `exporter.yml` into your Zeebe `application.yml` +1. Copy it somewhere accessible the Zeebe broker +1. Add the following to your Zeebe broker's `application.yaml` + +```yaml +zeebe: + broker: + exporters: + kafka: + className: io.zeebe.exporters.kafka.KafkaExporter + jarPath: /path/to/zeebe-kafka-exporter-3.0.0-jar-with-dependencies.jar + args: + producer: + # update this to point to your Kafka brokers + servers: "kafka-1:9092,kafka-2:9092" +``` + 1. Update the configuration's list of servers to point to your Kafka instances. The next time you start your Zeebe cluster, all event-type records will be exported to their -respective Kafka topics. +respective Kafka topics. See [exporter/exporter.yml](/exporter/exporter.yml) for a fully commented +configuration file example. -> NOTE: there's a bug in Zeebe which prevents this exporter from being loaded in an isolated way. -> See [this issue](https://github.com/camunda-cloud/zeebe/issues/4196) for more. When that is fixed, -> then the exporter should be loaded as any other external exporter. +> NOTE: pre Zeebe 1.0.1, there was a bug in Zeebe which prevented this exporter from being loaded in +> an isolated way. See [this issue](https://github.com/camunda-cloud/zeebe/issues/4196) for more. If +> that's the case for you, then you _must_ place this exporter in the `lib/` folder of your Zeebe +> broker directory - by default that would be `/usr/local/zeebe/lib`. It will then be automatically +> available on the classpath, and you can omit the `jarPath` configuration option. This is not +> recommended however, and if you're on Zeebe 1.0.1 or greater, you should use the method described +> above. ## Usage @@ -338,6 +357,9 @@ zeebe: exporters: kafka: className: io.zeebe.exporters.kafka.KafkaExporter + # Update this path to the location of the JAR + # Note that this must be visible to the broker process + jarPath: /path/to/zeebe-kafka-exporter-3.0.0-jar-with-dependencies.jar args: # Controls the number of records to buffer in a single record batch before forcing a flush. Note # that a flush may occur before anyway due to periodic flushing. This setting should help you diff --git a/exporter/exporter.yml b/exporter/exporter.yml index 7a4759d..cdcf77e 100644 --- a/exporter/exporter.yml +++ b/exporter/exporter.yml @@ -3,6 +3,9 @@ zeebe: exporters: kafka: className: io.zeebe.exporters.kafka.KafkaExporter + # Update this path to the location of the JAR + # Note that this must be visible to the broker process + jarPath: /path/to/zeebe-kafka-exporter-3.0.0-jar-with-dependencies.jar args: # Controls the number of records to buffer in a single record batch before forcing a flush. Note # that a flush may occur before anyway due to periodic flushing. This setting should help you