Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(fvt): tweak to work across more versions #2615

Merged
merged 2 commits into from
Aug 21, 2023
Merged

chore(fvt): tweak to work across more versions #2615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
270f507
chore(fvt): tweak to work across more versions
dnwe Aug 21, 2023
2b54832
fix(fvt): ensure correct version in consumer tests
dnwe Aug 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/fvt.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,18 @@ jobs:
matrix:
go-version: [1.21.x]
kafka-version: [3.3.2, 3.4.1, 3.5.1]
include:
- kafka-version: 3.3.2
scala-version: 2.13
- kafka-version: 3.4.1
scala-version: 2.13
- kafka-version: 3.5.1
scala-version: 2.13
env:
DEBUG: true
GOFLAGS: -trimpath
KAFKA_VERSION: ${{ matrix.kafka-version }}
SCALA_VERSION: ${{ matrix.scala-version }}
steps:
- uses: actions/checkout@v3
- name: Setup Docker
Expand Down
5 changes: 5 additions & 0 deletions Dockerfile.kafka
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ RUN cd /tmp \
&& for DIR in /opt/kafka-*; do cp -v jaxb-api-2.3.0.jar $DIR/libs/ ; done \
&& rm -f jaxb-api-2.3.0.jar

WORKDIR /opt/kafka-${KAFKA_VERSION}

ENV JAVA_MAJOR_VERSION=11

RUN sed -e "s/JAVA_MAJOR_VERSION=.*/JAVA_MAJOR_VERSION=${JAVA_MAJOR_VERSION}/" -i"" ./bin/kafka-run-class.sh

COPY entrypoint.sh /

Expand Down
44 changes: 24 additions & 20 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
version: '3.9'
services:
zookeeper-1:
hostname: 'zookeeper-1'
image: 'docker.io/library/zookeeper:3.6.3'
restart: always
environment:
Expand All @@ -12,6 +13,7 @@ services:
ZOO_MAX_CLIENT_CNXNS: '0'
ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
zookeeper-2:
hostname: 'zookeeper-2'
image: 'docker.io/library/zookeeper:3.6.3'
restart: always
environment:
Expand All @@ -23,6 +25,7 @@ services:
ZOO_MAX_CLIENT_CNXNS: '0'
ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
zookeeper-3:
hostname: 'zookeeper-3'
image: 'docker.io/library/zookeeper:3.6.3'
restart: always
environment:
Expand All @@ -34,6 +37,7 @@ services:
ZOO_MAX_CLIENT_CNXNS: '0'
ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
kafka-1:
hostname: 'kafka-1'
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
build:
context: .
Expand All @@ -45,12 +49,9 @@ services:
test:
[
'CMD',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-1:9091',
'--broker',
'1',
'--describe',
]
interval: 15s
timeout: 15s
Expand Down Expand Up @@ -78,7 +79,10 @@ services:
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-2:
hostname: 'kafka-2'
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
build:
context: .
Expand All @@ -90,12 +94,9 @@ services:
test:
[
'CMD',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-2:9091',
'--broker',
'2',
'--describe',
]
interval: 15s
timeout: 15s
Expand Down Expand Up @@ -123,7 +124,10 @@ services:
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-3:
hostname: 'kafka-3'
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
build:
context: .
Expand All @@ -135,12 +139,9 @@ services:
test:
[
'CMD',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-3:9091',
'--broker',
'3',
'--describe',
]
interval: 15s
timeout: 15s
Expand Down Expand Up @@ -168,7 +169,10 @@ services:
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-4:
hostname: 'kafka-4'
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
build:
context: .
Expand All @@ -180,12 +184,9 @@ services:
test:
[
'CMD',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-4:9091',
'--broker',
'4',
'--describe',
]
interval: 15s
timeout: 15s
Expand Down Expand Up @@ -213,7 +214,10 @@ services:
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-5:
hostname: 'kafka-5'
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
build:
context: .
Expand All @@ -225,12 +229,9 @@ services:
test:
[
'CMD',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-5:9091',
'--broker',
'5',
'--describe',
]
interval: 15s
timeout: 15s
Expand Down Expand Up @@ -258,7 +259,10 @@ services:
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
toxiproxy:
hostname: 'toxiproxy'
image: 'ghcr.io/shopify/toxiproxy:2.4.0'
healthcheck:
test: ['CMD', '/toxiproxy-cli', 'l']
Expand Down
4 changes: 3 additions & 1 deletion entrypoint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ fi
cd "${KAFKA_HOME}" || exit 1

# discard all empty/commented lines
sed -e '/^#/d' -e '/^$/d' -i"" config/server.properties
sed -e '/^#/d' -e '/^$/d' -i".orig" config/server.properties

echo "########################################################################" >>config/server.properties

# emulate kafka_configure_from_environment_variables from bitnami/bitnami-docker-kafka
for var in "${!KAFKA_CFG_@}"; do
Expand Down
8 changes: 6 additions & 2 deletions functional_consumer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,10 @@ func TestConsumerHighWaterMarkOffset(t *testing.T) {
setupFunctionalTest(t)
defer teardownFunctionalTest(t)

p, err := NewSyncProducer(FunctionalTestEnv.KafkaBrokerAddrs, nil)
config := NewFunctionalTestConfig()
config.Producer.Return.Successes = true

p, err := NewSyncProducer(FunctionalTestEnv.KafkaBrokerAddrs, config)
if err != nil {
t.Fatal(err)
}
Expand All @@ -57,7 +60,7 @@ func TestConsumerHighWaterMarkOffset(t *testing.T) {
t.Fatal(err)
}

c, err := NewConsumer(FunctionalTestEnv.KafkaBrokerAddrs, NewFunctionalTestConfig())
c, err := NewConsumer(FunctionalTestEnv.KafkaBrokerAddrs, config)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -127,6 +130,7 @@ func TestVersionMatrixLZ4(t *testing.T) {

// Support for zstd codec was introduced in v2.1.0.0
func TestVersionMatrixZstd(t *testing.T) {
checkKafkaVersion(t, "2.1.0")
metrics.UseNilMetrics = true // disable Sarama's go-metrics library
t.Cleanup(func() {
metrics.UseNilMetrics = false
Expand Down
Loading