chore: add newer kafka versions and bump Go in CI (#2969)

- bump Go versions used in CI to baseline of 1.20.x and latest 1.23.x - add newer Kafka version constants to utils.go - bring newer Kafka versions into the FVT rotation and bump default to 3.6.2 in Dockerfile and docker-compose - use newer ZooKeeper version in docker-compose too - bump golangci-lint version to latest (and fix for vet changes) Signed-off-by: Dominic Evans <dominic.evans@uk.ibm.com>
IBM · Aug 21, 2024 · 98935c0 · 98935c0
1 parent c6b288a
commit 98935c0
Show file tree

Hide file tree

Showing 14 changed files with 65 additions and 45 deletions.
diff --git a/.github/workflows/apidiff.yml b/.github/workflows/apidiff.yml
@@ -23,7 +23,7 @@ jobs:
  - name: Setup Go
  uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
  with:
- go-version: 1.22.x
+ go-version: 1.23.x
  - name: Add GOBIN to PATH
  run: echo "$(go env GOPATH)/bin" >>$GITHUB_PATH
  - name: Install apidiff cmd

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- go-version: [1.22.x]
+ go-version: [1.23.x]
  steps:
  - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
@@ -39,14 +39,14 @@ jobs:
  GOFLAGS: -tags=functional
  uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 # v6.1.0
  with:
- version: v1.57.2
+ version: v1.60.2
  test:
  name: Unit Testing with Go ${{ matrix.go-version }}
  runs-on: ubuntu-latest
  strategy:
  fail-fast: false
  matrix:
- go-version: [1.19.x, 1.21.x, 1.22.x]
+ go-version: [1.20.x, 1.23.x]
  env:
  DEBUG: true
  GOFLAGS: -trimpath

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -37,7 +37,7 @@ jobs:
  - name: Setup Go
  uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
  with:
- go-version: 1.22.x
+ go-version: 1.23.x
  - name: Autobuild
  uses: github/codeql-action/autobuild@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0
  - name: Perform CodeQL Analysis

diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml
@@ -28,6 +28,6 @@ jobs:
  - name: Setup Go
  uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
  with:
- go-version: 1.22.x
+ go-version: 1.23.x
  - name: Run any fuzzing tests
  run: go test -list . | grep '^Fuzz' | parallel 'go test -v -run=^{}$ -fuzz=^{}$ -fuzztime=5m'
diff --git a/.github/workflows/fvt-main.yml b/.github/workflows/fvt-main.yml
@@ -16,8 +16,8 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- go-version: [1.22.x]
- kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.6.0]
+ go-version: [1.23.x]
+ kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.6.2, 3.8.0]
  include:
  - kafka-version: 1.0.2
  scala-version: 2.11
@@ -33,7 +33,9 @@ jobs:
  scala-version: 2.12
  - kafka-version: 3.3.2
  scala-version: 2.13
- - kafka-version: 3.6.0
+ - kafka-version: 3.6.2
+ scala-version: 2.13
+ - kafka-version: 3.8.0
  scala-version: 2.13
  uses: ./.github/workflows/fvt.yml
  with:

diff --git a/.github/workflows/fvt-pr.yml b/.github/workflows/fvt-pr.yml
@@ -15,14 +15,16 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- go-version: [1.22.x]
- kafka-version: [1.0.2, 2.6.2, 3.6.0]
+ go-version: [1.23.x]
+ kafka-version: [1.0.2, 2.6.2, 3.6.2, 3.8.0]
  include:
  - kafka-version: 1.0.2
  scala-version: 2.11
  - kafka-version: 2.6.2
  scala-version: 2.12
- - kafka-version: 3.6.0
+ - kafka-version: 3.6.2
+ scala-version: 2.13
+ - kafka-version: 3.8.0
  scala-version: 2.13
  uses: ./.github/workflows/fvt.yml
  with:

diff --git a/.github/workflows/fvt.yml b/.github/workflows/fvt.yml
@@ -5,11 +5,11 @@ on:
  go-version:
  required: false
  type: string
- default: 1.22.x
+ default: 1.23.x
  kafka-version:
  required: false
  type: string
- default: 3.6.0
+ default: 3.6.2
  scala-version:
  required: false
  type: string

diff --git a/.github/workflows/i386.yml b/.github/workflows/i386.yml
@@ -28,7 +28,7 @@ jobs:
  - name: Setup Go
  uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
  with:
- go-version: 1.22.x
+ go-version: 1.23.x
  - name: staticcheck
  env:
  GOARCH: 386

diff --git a/.golangci.yml b/.golangci.yml
@@ -1,4 +1,5 @@
 run:
+ go: "1.20"
  timeout: 5m
  deadline: 10m
 
@@ -57,7 +58,7 @@ linters:
  enable:
  - bodyclose
  - depguard
- - exportloopref
+ # - copyloopvar
  - dogsled
  - errcheck
  - errorlint
@@ -79,6 +80,7 @@ linters:
 
 issues:
  exclude:
+ - "G115: integer overflow conversion"
  - "G404: Use of weak random number generator"
  exclude-rules:
  # exclude some linters from running on certains files.

diff --git a/Dockerfile.kafka b/Dockerfile.kafka
@@ -19,7 +19,7 @@ RUN cd /etc/java/java-11-openjdk/*/conf/security \
 ARG SCALA_VERSION="2.13"
 ARG KAFKA_VERSION="3.6.0"
 
-# https://github.com/apache/kafka/blob/9989b68d0d38c8f1357f78bf9d53a58c1476188d/tests/docker/Dockerfile#L46-L72
+# https://github.com/apache/kafka/blob/0eaaff88cf68bc2c24d4874ff9bc1cc2b493c24b/tests/docker/Dockerfile#L75-L103
 ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages"
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN mkdir -p "/opt/kafka-${KAFKA_VERSION}" \

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,7 @@
 services:
  zookeeper-1:
  hostname: 'zookeeper-1'
- image: 'docker.io/library/zookeeper:3.6.3'
+ image: 'docker.io/library/zookeeper:3.7.2'
  init: true
  restart: always
  environment:
@@ -14,7 +14,7 @@ services:
  ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
  zookeeper-2:
  hostname: 'zookeeper-2'
- image: 'docker.io/library/zookeeper:3.6.3'
+ image: 'docker.io/library/zookeeper:3.7.2'
  init: true
  restart: always
  environment:
@@ -27,7 +27,7 @@ services:
  ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
  zookeeper-3:
  hostname: 'zookeeper-3'
- image: 'docker.io/library/zookeeper:3.6.3'
+ image: 'docker.io/library/zookeeper:3.7.2'
  init: true
  restart: always
  environment:
@@ -40,19 +40,19 @@ services:
  ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
  kafka-1:
  hostname: 'kafka-1'
- image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.2}'
  init: true
  build:
  context: .
  dockerfile: Dockerfile.kafka
  args:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2}
  SCALA_VERSION: ${SCALA_VERSION:-2.13}
  healthcheck:
  test:
  [
  'CMD',
- '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
+ '/opt/kafka-${KAFKA_VERSION:-3.6.2}/bin/kafka-broker-api-versions.sh',
  '--bootstrap-server',
  'kafka-1:9091',
  ]
@@ -67,7 +67,7 @@ services:
  - toxiproxy
  restart: always
  environment:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2}
  KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
  KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
  KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
@@ -86,19 +86,19 @@ services:
  KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
  kafka-2:
  hostname: 'kafka-2'
- image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.2}'
  init: true
  build:
  context: .
  dockerfile: Dockerfile.kafka
  args:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2}
  SCALA_VERSION: ${SCALA_VERSION:-2.13}
  healthcheck:
  test:
  [
  'CMD',
- '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
+ '/opt/kafka-${KAFKA_VERSION:-3.6.2}/bin/kafka-broker-api-versions.sh',
  '--bootstrap-server',
  'kafka-2:9091',
  ]
@@ -113,7 +113,7 @@ services:
  - toxiproxy
  restart: always
  environment:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2}
  KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
  KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
  KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
@@ -132,19 +132,19 @@ services:
  KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
  kafka-3:
  hostname: 'kafka-3'
- image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.2}'
  init: true
  build:
  context: .
  dockerfile: Dockerfile.kafka
  args:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2}
  SCALA_VERSION: ${SCALA_VERSION:-2.13}
  healthcheck:
  test:
  [
  'CMD',
- '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
+ '/opt/kafka-${KAFKA_VERSION:-3.6.2}/bin/kafka-broker-api-versions.sh',
  '--bootstrap-server',
  'kafka-3:9091',
  ]
@@ -159,7 +159,7 @@ services:
  - toxiproxy
  restart: always
  environment:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2}
  KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
  KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
  KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
@@ -178,19 +178,19 @@ services:
  KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
  kafka-4:
  hostname: 'kafka-4'
- image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.2}'
  init: true
  build:
  context: .
  dockerfile: Dockerfile.kafka
  args:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2}
  SCALA_VERSION: ${SCALA_VERSION:-2.13}
  healthcheck:
  test:
  [
  'CMD',
- '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
+ '/opt/kafka-${KAFKA_VERSION:-3.6.2}/bin/kafka-broker-api-versions.sh',
  '--bootstrap-server',
  'kafka-4:9091',
  ]
@@ -205,7 +205,7 @@ services:
  - toxiproxy
  restart: always
  environment:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2}
  KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
  KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
  KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
@@ -224,19 +224,19 @@ services:
  KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
  kafka-5:
  hostname: 'kafka-5'
- image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.2}'
  init: true
  build:
  context: .
  dockerfile: Dockerfile.kafka
  args:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2}
  SCALA_VERSION: ${SCALA_VERSION:-2.13}
  healthcheck:
  test:
  [
  'CMD',
- '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
+ '/opt/kafka-${KAFKA_VERSION:-3.6.2}/bin/kafka-broker-api-versions.sh',
  '--bootstrap-server',
  'kafka-5:9091',
  ]
@@ -251,7 +251,7 @@ services:
  - toxiproxy
  restart: always
  environment:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.2}
  KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
  KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
  KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'

diff --git a/entrypoint.sh b/entrypoint.sh
@@ -3,7 +3,7 @@
 set -eu
 set -o pipefail
 
-KAFKA_VERSION="${KAFKA_VERSION:-3.6.0}"
+KAFKA_VERSION="${KAFKA_VERSION:-3.6.2}"
 KAFKA_HOME="/opt/kafka-${KAFKA_VERSION}"
 
 if [ ! -d "${KAFKA_HOME}" ]; then

diff --git a/record_test.go b/record_test.go
@@ -248,7 +248,7 @@ func TestRecordBatchDecoding(t *testing.T) {
  // anyway. We only set it here to ensure that comparison succeeds.
  batch.CompressionLevel = tc.batch.CompressionLevel
  if !reflect.DeepEqual(batch, tc.batch) {
- t.Errorf(spew.Sprintf("invalid decode of %s\ngot %+v\nwanted %+v", tc.name, batch, tc.batch))
+ t.Error(spew.Sprintf("invalid decode of %s\ngot %+v\nwanted %+v", tc.name, batch, tc.batch))
  }
  }
 }