From 11752540e6cf701dbd0c9e948287ef43aed79c9e Mon Sep 17 00:00:00 2001 From: Lucas Kent Date: Tue, 20 Feb 2024 10:35:37 +1100 Subject: [PATCH] Update kafka image to 3.6.1 --- .../benches/windsock/kafka/bench.rs | 22 ++++++++++++++++--- .../kafka/bench/docker-compose.yaml | 12 +++++++--- .../kafka/cluster/docker-compose.yaml | 17 ++++++++------ .../kafka/passthrough/docker-compose.yaml | 14 ++++++++---- test-helpers/src/docker_compose.rs | 2 +- 5 files changed, 49 insertions(+), 18 deletions(-) diff --git a/shotover-proxy/benches/windsock/kafka/bench.rs b/shotover-proxy/benches/windsock/kafka/bench.rs index 003035fd5..6b9b7b5d1 100644 --- a/shotover-proxy/benches/windsock/kafka/bench.rs +++ b/shotover-proxy/benches/windsock/kafka/bench.rs @@ -124,16 +124,32 @@ impl KafkaBench { tasks.push(tokio::spawn(async move { node.run_container( - "bitnami/kafka:3.4.0-debian-11-r22", + "bitnami/kafka:3.6.1-debian-11-r24", &[ ("ALLOW_PLAINTEXT_LISTENER".to_owned(), "yes".to_owned()), ( "KAFKA_CFG_ADVERTISED_LISTENERS".to_owned(), - format!("PLAINTEXT://{ip}:{port}"), + format!("BROKER://{ip}:{port}"), ), ( "KAFKA_CFG_LISTENERS".to_owned(), - format!("PLAINTEXT://:{port},CONTROLLER://:9093"), + format!("BROKER://:{port},CONTROLLER://:9093"), + ), + ( + "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP".to_owned(), + "CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT".to_owned(), + ), + ( + "KAFKA_CFG_INTER_BROKER_LISTENER_NAME".to_owned(), + "BROKER".to_owned(), + ), + ( + "KAFKA_CFG_CONTROLLER_LISTENER_NAMES".to_owned(), + "CONTROLLER".to_owned(), + ), + ( + "KAFKA_CFG_PROCESS_ROLES".to_owned(), + "controller,broker".to_owned(), ), ( "KAFKA_HEAP_OPTS".to_owned(), diff --git a/shotover-proxy/tests/test-configs/kafka/bench/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/bench/docker-compose.yaml index c33b3a8e7..aaf57730e 100644 --- a/shotover-proxy/tests/test-configs/kafka/bench/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/bench/docker-compose.yaml @@ -1,10 +1,16 @@ version: "3" services: kafka: - image: 'bitnami/kafka:3.4.0-debian-11-r22' + image: 'bitnami/kafka:3.6.1-debian-11-r24' ports: - '9192:9192' environment: - - KAFKA_CFG_LISTENERS=PLAINTEXT://:9192,CONTROLLER://:9093 - - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9192 + - KAFKA_CFG_LISTENERS=BROKER://:9192,CONTROLLER://:9093 + - KAFKA_CFG_ADVERTISED_LISTENERS=BROKER://127.0.0.1:9192 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=BROKER + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_CFG_PROCESS_ROLES=controller,broker + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka0:9093 + - KAFKA_CFG_NODE_ID=0 - ALLOW_PLAINTEXT_LISTENER=yes diff --git a/shotover-proxy/tests/test-configs/kafka/cluster/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster/docker-compose.yaml index b7a3fe4a7..8a84841e9 100644 --- a/shotover-proxy/tests/test-configs/kafka/cluster/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/cluster/docker-compose.yaml @@ -10,17 +10,20 @@ networks: gateway: 172.16.1.1 services: kafka0: - image: &image 'bitnami/kafka:3.4.0-debian-11-r22' + image: &image 'bitnami/kafka:3.6.1-debian-11-r24' networks: cluster_subnet: ipv4_address: 172.16.1.2 environment: &environment - KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093" - KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://172.16.1.2:9092" - ALLOW_PLAINTEXT_LISTENER: "yes" - KAFKA_KRAFT_CLUSTER_ID: "abcdefghijklmnopqrstuv" + KAFKA_CFG_LISTENERS: "BROKER://:9092,CONTROLLER://:9093" + KAFKA_CFG_ADVERTISED_LISTENERS: "BROKER://172.16.1.2:9092" + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT" + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "BROKER" + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER" + KAFKA_CFG_PROCESS_ROLES: "controller,broker" KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "0@kafka0:9093,1@kafka1:9093,2@kafka2:9093" KAFKA_CFG_NODE_ID: 0 + ALLOW_PLAINTEXT_LISTENER: "yes" volumes: &volumes - type: tmpfs target: /bitnami/kafka @@ -31,7 +34,7 @@ services: ipv4_address: 172.16.1.3 environment: <<: *environment - KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://172.16.1.3:9092" + KAFKA_CFG_ADVERTISED_LISTENERS: "BROKER://172.16.1.3:9092" KAFKA_CFG_NODE_ID: 1 volumes: *volumes kafka2: @@ -41,6 +44,6 @@ services: ipv4_address: 172.16.1.4 environment: <<: *environment - KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://172.16.1.4:9092" + KAFKA_CFG_ADVERTISED_LISTENERS: "BROKER://172.16.1.4:9092" KAFKA_CFG_NODE_ID: 2 volumes: *volumes diff --git a/shotover-proxy/tests/test-configs/kafka/passthrough/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/passthrough/docker-compose.yaml index 6330ef5e9..62707d6c9 100644 --- a/shotover-proxy/tests/test-configs/kafka/passthrough/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/passthrough/docker-compose.yaml @@ -1,12 +1,18 @@ version: "3" services: - kafka: - image: 'bitnami/kafka:3.4.0-debian-11-r22' + kafka0: + image: 'bitnami/kafka:3.6.1-debian-11-r24' ports: - '9092:9092' environment: - - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 - - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 + - KAFKA_CFG_LISTENERS=BROKER://:9092,CONTROLLER://:9093 + - KAFKA_CFG_ADVERTISED_LISTENERS=BROKER://127.0.0.1:9092 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=BROKER + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_CFG_PROCESS_ROLES=controller,broker + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka0:9093 + - KAFKA_CFG_NODE_ID=0 - ALLOW_PLAINTEXT_LISTENER=yes volumes: - type: tmpfs diff --git a/test-helpers/src/docker_compose.rs b/test-helpers/src/docker_compose.rs index ca8354cdf..fb6a0f747 100644 --- a/test-helpers/src/docker_compose.rs +++ b/test-helpers/src/docker_compose.rs @@ -71,7 +71,7 @@ pub static IMAGE_WAITERS: [Image; 11] = [ timeout: Duration::from_secs(120), }, Image { - name: "bitnami/kafka:3.4.0-debian-11-r22", + name: "bitnami/kafka:3.6.1-debian-11-r24", log_regex_to_wait_for: r"Kafka Server started", timeout: Duration::from_secs(120), },