From 98935c0268bd11b16354bbc718f7e7c5b9137686 Mon Sep 17 00:00:00 2001 From: Dominic Evans <8060970+dnwe@users.noreply.github.com> Date: Wed, 21 Aug 2024 20:03:47 +0100 Subject: [PATCH] chore: add newer kafka versions and bump Go in CI (#2969) - bump Go versions used in CI to baseline of 1.20.x and latest 1.23.x - add newer Kafka version constants to utils.go - bring newer Kafka versions into the FVT rotation and bump default to 3.6.2 in Dockerfile and docker-compose - use newer ZooKeeper version in docker-compose too - bump golangci-lint version to latest (and fix for vet changes) Signed-off-by: Dominic Evans --- .github/workflows/apidiff.yml | 2 +- .github/workflows/ci.yml | 6 ++-- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/fuzz.yml | 2 +- .github/workflows/fvt-main.yml | 8 +++-- .github/workflows/fvt-pr.yml | 8 +++-- .github/workflows/fvt.yml | 4 +-- .github/workflows/i386.yml | 2 +- .golangci.yml | 4 ++- Dockerfile.kafka | 2 +- docker-compose.yml | 46 +++++++++++++-------------- entrypoint.sh | 2 +- record_test.go | 2 +- utils.go | 20 ++++++++++-- 14 files changed, 65 insertions(+), 45 deletions(-) diff --git a/.github/workflows/apidiff.yml b/.github/workflows/apidiff.yml index 40cd51801..785d70bce 100644 --- a/.github/workflows/apidiff.yml +++ b/.github/workflows/apidiff.yml @@ -23,7 +23,7 @@ jobs: - name: Setup Go uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: - go-version: 1.22.x + go-version: 1.23.x - name: Add GOBIN to PATH run: echo "$(go env GOPATH)/bin" >>$GITHUB_PATH - name: Install apidiff cmd diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15404b609..cc07ceda6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - go-version: [1.22.x] + go-version: [1.23.x] steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: @@ -39,14 +39,14 @@ jobs: GOFLAGS: -tags=functional uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 # v6.1.0 with: - version: v1.57.2 + version: v1.60.2 test: name: Unit Testing with Go ${{ matrix.go-version }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - go-version: [1.19.x, 1.21.x, 1.22.x] + go-version: [1.20.x, 1.23.x] env: DEBUG: true GOFLAGS: -trimpath diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d0b97f56c..c6936f6a4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,7 +37,7 @@ jobs: - name: Setup Go uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: - go-version: 1.22.x + go-version: 1.23.x - name: Autobuild uses: github/codeql-action/autobuild@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 - name: Perform CodeQL Analysis diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 9d13ee1fe..36bef06f3 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -28,6 +28,6 @@ jobs: - name: Setup Go uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: - go-version: 1.22.x + go-version: 1.23.x - name: Run any fuzzing tests run: go test -list . | grep '^Fuzz' | parallel 'go test -v -run=^{}$ -fuzz=^{}$ -fuzztime=5m' diff --git a/.github/workflows/fvt-main.yml b/.github/workflows/fvt-main.yml index 8f6d36580..fae7c46a7 100644 --- a/.github/workflows/fvt-main.yml +++ b/.github/workflows/fvt-main.yml @@ -16,8 +16,8 @@ jobs: strategy: fail-fast: false matrix: - go-version: [1.22.x] - kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.6.0] + go-version: [1.23.x] + kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.6.2, 3.8.0] include: - kafka-version: 1.0.2 scala-version: 2.11 @@ -33,7 +33,9 @@ jobs: scala-version: 2.12 - kafka-version: 3.3.2 scala-version: 2.13 - - kafka-version: 3.6.0 + - kafka-version: 3.6.2 + scala-version: 2.13 + - kafka-version: 3.8.0 scala-version: 2.13 uses: ./.github/workflows/fvt.yml with: diff --git a/.github/workflows/fvt-pr.yml b/.github/workflows/fvt-pr.yml index 6997095cc..2c8824bf5 100644 --- a/.github/workflows/fvt-pr.yml +++ b/.github/workflows/fvt-pr.yml @@ -15,14 +15,16 @@ jobs: strategy: fail-fast: false matrix: - go-version: [1.22.x] - kafka-version: [1.0.2, 2.6.2, 3.6.0] + go-version: [1.23.x] + kafka-version: [1.0.2, 2.6.2, 3.6.2, 3.8.0] include: - kafka-version: 1.0.2 scala-version: 2.11 - kafka-version: 2.6.2 scala-version: 2.12 - - kafka-version: 3.6.0 + - kafka-version: 3.6.2 + scala-version: 2.13 + - kafka-version: 3.8.0 scala-version: 2.13 uses: ./.github/workflows/fvt.yml with: diff --git a/.github/workflows/fvt.yml b/.github/workflows/fvt.yml index c45bfb6ad..4537a51af 100644 --- a/.github/workflows/fvt.yml +++ b/.github/workflows/fvt.yml @@ -5,11 +5,11 @@ on: go-version: required: false type: string - default: 1.22.x + default: 1.23.x kafka-version: required: false type: string - default: 3.6.0 + default: 3.6.2 scala-version: required: false type: string diff --git a/.github/workflows/i386.yml b/.github/workflows/i386.yml index fa03eda74..43738c9b2 100644 --- a/.github/workflows/i386.yml +++ b/.github/workflows/i386.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Go uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: - go-version: 1.22.x + go-version: 1.23.x - name: staticcheck env: GOARCH: 386 diff --git a/.golangci.yml b/.golangci.yml index 72e3e4c24..2e029401d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,4 +1,5 @@ run: + go: "1.20" timeout: 5m deadline: 10m @@ -57,7 +58,7 @@ linters: enable: - bodyclose - depguard - - exportloopref + # - copyloopvar - dogsled - errcheck - errorlint @@ -79,6 +80,7 @@ linters: issues: exclude: + - "G115: integer overflow conversion" - "G404: Use of weak random number generator" exclude-rules: # exclude some linters from running on certains files. diff --git a/Dockerfile.kafka b/Dockerfile.kafka index 40f5f333b..8fae4bed4 100644 --- a/Dockerfile.kafka +++ b/Dockerfile.kafka @@ -19,7 +19,7 @@ RUN cd /etc/java/java-11-openjdk/*/conf/security \ ARG SCALA_VERSION="2.13" ARG KAFKA_VERSION="3.6.0" -# https://github.com/apache/kafka/blob/9989b68d0d38c8f1357f78bf9d53a58c1476188d/tests/docker/Dockerfile#L46-L72 +# https://github.com/apache/kafka/blob/0eaaff88cf68bc2c24d4874ff9bc1cc2b493c24b/tests/docker/Dockerfile#L75-L103 ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages" SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN mkdir -p "/opt/kafka-${KAFKA_VERSION}" \ diff --git a/docker-compose.yml b/docker-compose.yml index 204768e32..a0e3d2e21 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: zookeeper-1: hostname: 'zookeeper-1' - image: 'docker.io/library/zookeeper:3.6.3' + image: 'docker.io/library/zookeeper:3.7.2' init: true restart: always environment: @@ -14,7 +14,7 @@ services: ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' zookeeper-2: hostname: 'zookeeper-2' - image: 'docker.io/library/zookeeper:3.6.3' + image: 'docker.io/library/zookeeper:3.7.2' init: true restart: always environment: @@ -27,7 +27,7 @@ services: ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' zookeeper-3: hostname: 'zookeeper-3' - image: 'docker.io/library/zookeeper:3.6.3' + image: 'docker.io/library/zookeeper:3.7.2' init: true restart: always environment: @@ -40,19 +40,19 @@ services: ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' kafka-1: hostname: 'kafka-1' - image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.2}' init: true build: context: . dockerfile: Dockerfile.kafka args: - KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0} + KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2} SCALA_VERSION: ${SCALA_VERSION:-2.13} healthcheck: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh', + '/opt/kafka-${KAFKA_VERSION:-3.6.2}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-1:9091', ] @@ -67,7 +67,7 @@ services: - toxiproxy restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0} + KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091' @@ -86,19 +86,19 @@ services: KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-2: hostname: 'kafka-2' - image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.2}' init: true build: context: . dockerfile: Dockerfile.kafka args: - KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0} + KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2} SCALA_VERSION: ${SCALA_VERSION:-2.13} healthcheck: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh', + '/opt/kafka-${KAFKA_VERSION:-3.6.2}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-2:9091', ] @@ -113,7 +113,7 @@ services: - toxiproxy restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0} + KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092' @@ -132,19 +132,19 @@ services: KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-3: hostname: 'kafka-3' - image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.2}' init: true build: context: . dockerfile: Dockerfile.kafka args: - KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0} + KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2} SCALA_VERSION: ${SCALA_VERSION:-2.13} healthcheck: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh', + '/opt/kafka-${KAFKA_VERSION:-3.6.2}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-3:9091', ] @@ -159,7 +159,7 @@ services: - toxiproxy restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0} + KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093' @@ -178,19 +178,19 @@ services: KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-4: hostname: 'kafka-4' - image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.2}' init: true build: context: . dockerfile: Dockerfile.kafka args: - KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0} + KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2} SCALA_VERSION: ${SCALA_VERSION:-2.13} healthcheck: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh', + '/opt/kafka-${KAFKA_VERSION:-3.6.2}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-4:9091', ] @@ -205,7 +205,7 @@ services: - toxiproxy restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0} + KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094' @@ -224,19 +224,19 @@ services: KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-5: hostname: 'kafka-5' - image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.2}' init: true build: context: . dockerfile: Dockerfile.kafka args: - KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0} + KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2} SCALA_VERSION: ${SCALA_VERSION:-2.13} healthcheck: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh', + '/opt/kafka-${KAFKA_VERSION:-3.6.2}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-5:9091', ] @@ -251,7 +251,7 @@ services: - toxiproxy restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0} + KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095' diff --git a/entrypoint.sh b/entrypoint.sh index 9fe9a44b1..516a8dc38 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,7 +3,7 @@ set -eu set -o pipefail -KAFKA_VERSION="${KAFKA_VERSION:-3.6.0}" +KAFKA_VERSION="${KAFKA_VERSION:-3.6.2}" KAFKA_HOME="/opt/kafka-${KAFKA_VERSION}" if [ ! -d "${KAFKA_HOME}" ]; then diff --git a/record_test.go b/record_test.go index 960ed733b..aec5c848d 100644 --- a/record_test.go +++ b/record_test.go @@ -248,7 +248,7 @@ func TestRecordBatchDecoding(t *testing.T) { // anyway. We only set it here to ensure that comparison succeeds. batch.CompressionLevel = tc.batch.CompressionLevel if !reflect.DeepEqual(batch, tc.batch) { - t.Errorf(spew.Sprintf("invalid decode of %s\ngot %+v\nwanted %+v", tc.name, batch, tc.batch)) + t.Error(spew.Sprintf("invalid decode of %s\ngot %+v\nwanted %+v", tc.name, batch, tc.batch)) } } } diff --git a/utils.go b/utils.go index feadc0065..be5fe555d 100644 --- a/utils.go +++ b/utils.go @@ -198,7 +198,13 @@ var ( V3_4_1_0 = newKafkaVersion(3, 4, 1, 0) V3_5_0_0 = newKafkaVersion(3, 5, 0, 0) V3_5_1_0 = newKafkaVersion(3, 5, 1, 0) + V3_5_2_0 = newKafkaVersion(3, 5, 2, 0) V3_6_0_0 = newKafkaVersion(3, 6, 0, 0) + V3_6_1_0 = newKafkaVersion(3, 6, 1, 0) + V3_6_2_0 = newKafkaVersion(3, 6, 2, 0) + V3_7_0_0 = newKafkaVersion(3, 7, 0, 0) + V3_7_1_0 = newKafkaVersion(3, 7, 1, 0) + V3_8_0_0 = newKafkaVersion(3, 8, 0, 0) SupportedVersions = []KafkaVersion{ V0_8_2_0, @@ -237,8 +243,10 @@ var ( V2_6_0_0, V2_6_1_0, V2_6_2_0, + V2_6_3_0, V2_7_0_0, V2_7_1_0, + V2_7_2_0, V2_8_0_0, V2_8_1_0, V2_8_2_0, @@ -259,10 +267,16 @@ var ( V3_4_1_0, V3_5_0_0, V3_5_1_0, + V3_5_2_0, V3_6_0_0, + V3_6_1_0, + V3_6_2_0, + V3_7_0_0, + V3_7_1_0, + V3_8_0_0, } MinVersion = V0_8_2_0 - MaxVersion = V3_6_0_0 + MaxVersion = V3_8_0_0 DefaultVersion = V2_1_0_0 // reduced set of protocol versions to matrix test @@ -274,11 +288,11 @@ var ( V2_0_1_0, V2_2_2_0, V2_4_1_0, - V2_6_2_0, + V2_6_3_0, V2_8_2_0, V3_1_2_0, V3_3_2_0, - V3_6_0_0, + V3_6_2_0, } )