Merge branch 'main' into fix-metadata-full-false

IBM · Sep 10, 2024 · f4f143e · f4f143e
2 parents da5c239 + 4db4716
commit f4f143e
Show file tree

Hide file tree

Showing 35 changed files with 260 additions and 112 deletions.
diff --git a/.github/workflows/apidiff.yml b/.github/workflows/apidiff.yml
@@ -23,13 +23,13 @@ jobs:
  - name: Setup Go
  uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
  with:
- go-version: 1.22.x
+ go-version: 1.23.x
  - name: Add GOBIN to PATH
  run: echo "$(go env GOPATH)/bin" >>$GITHUB_PATH
  - name: Install apidiff cmd
  run: go install golang.org/x/exp/cmd/apidiff@v0.0.0-20231006140011-7918f672742d
  - name: Checkout base code
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+ uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
  ref: ${{ github.base_ref }}
  path: "base"
@@ -38,7 +38,7 @@ jobs:
  run: apidiff -m -w ../baseline.bin .
  working-directory: "base"
  - name: Checkout updated code
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+ uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
  path: "updated"
  persist-credentials: false

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,9 +25,9 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- go-version: [1.22.x]
+ go-version: [1.23.x]
  steps:
- - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
  persist-credentials: false
  - name: Setup Go
@@ -37,21 +37,21 @@ jobs:
  - name: golangci-lint
  env:
  GOFLAGS: -tags=functional
- uses: golangci/golangci-lint-action@82d40c283aeb1f2b6595839195e95c2d6a49081b # v5.0.0
+ uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 # v6.1.0
  with:
- version: v1.57.2
+ version: v1.60.2
  test:
  name: Unit Testing with Go ${{ matrix.go-version }}
  runs-on: ubuntu-latest
  strategy:
  fail-fast: false
  matrix:
- go-version: [1.19.x, 1.21.x, 1.22.x]
+ go-version: [1.20.x, 1.23.x]
  env:
  DEBUG: true
  GOFLAGS: -trimpath
  steps:
- - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
  persist-credentials: false
  - name: Setup Go

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -27,18 +27,18 @@ jobs:
  language: ["go"]
  steps:
  - name: Checkout repository
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+ uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
  persist-credentials: false
  - name: Initialize CodeQL
- uses: github/codeql-action/init@8f596b4ae3cb3c588a5c46780b86dd53fef16c52 # v3.25.2
+ uses: github/codeql-action/init@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0
  with:
  languages: ${{ matrix.language }}
  - name: Setup Go
  uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
  with:
- go-version: 1.22.x
+ go-version: 1.23.x
  - name: Autobuild
- uses: github/codeql-action/autobuild@8f596b4ae3cb3c588a5c46780b86dd53fef16c52 # v3.25.2
+ uses: github/codeql-action/autobuild@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0
  - name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@8f596b4ae3cb3c588a5c46780b86dd53fef16c52 # v3.25.2
+ uses: github/codeql-action/analyze@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0
diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
@@ -22,7 +22,7 @@ jobs:
  runs-on: ubuntu-latest
  steps:
  - name: 'Checkout Repository'
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+ uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
  persist-credentials: false
  - name: 'Dependency Review'

diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml
@@ -22,12 +22,12 @@ jobs:
  env:
  GOFLAGS: -trimpath
  steps:
- - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
  persist-credentials: false
  - name: Setup Go
  uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
  with:
- go-version: 1.22.x
+ go-version: 1.23.x
  - name: Run any fuzzing tests
  run: go test -list . | grep '^Fuzz' | parallel 'go test -v -run=^{}$ -fuzz=^{}$ -fuzztime=5m'
diff --git a/.github/workflows/fvt-main.yml b/.github/workflows/fvt-main.yml
@@ -16,8 +16,8 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- go-version: [1.22.x]
- kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.6.0]
+ go-version: [1.23.x]
+ kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.6.2, 3.8.0]
  include:
  - kafka-version: 1.0.2
  scala-version: 2.11
@@ -33,7 +33,9 @@ jobs:
  scala-version: 2.12
  - kafka-version: 3.3.2
  scala-version: 2.13
- - kafka-version: 3.6.0
+ - kafka-version: 3.6.2
+ scala-version: 2.13
+ - kafka-version: 3.8.0
  scala-version: 2.13
  uses: ./.github/workflows/fvt.yml
  with:

diff --git a/.github/workflows/fvt-pr.yml b/.github/workflows/fvt-pr.yml
@@ -15,14 +15,16 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- go-version: [1.22.x]
- kafka-version: [1.0.2, 2.6.2, 3.6.0]
+ go-version: [1.23.x]
+ kafka-version: [1.0.2, 2.6.2, 3.6.2, 3.8.0]
  include:
  - kafka-version: 1.0.2
  scala-version: 2.11
  - kafka-version: 2.6.2
  scala-version: 2.12
- - kafka-version: 3.6.0
+ - kafka-version: 3.6.2
+ scala-version: 2.13
+ - kafka-version: 3.8.0
  scala-version: 2.13
  uses: ./.github/workflows/fvt.yml
  with:

diff --git a/.github/workflows/fvt.yml b/.github/workflows/fvt.yml
@@ -5,11 +5,11 @@ on:
  go-version:
  required: false
  type: string
- default: 1.22.x
+ default: 1.23.x
  kafka-version:
  required: false
  type: string
- default: 3.6.0
+ default: 3.6.2
  scala-version:
  required: false
  type: string
@@ -28,11 +28,11 @@ jobs:
  KAFKA_VERSION: ${{ inputs.kafka-version }}
  SCALA_VERSION: ${{ inputs.scala-version }}
  steps:
- - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
  persist-credentials: false
  - name: Setup Docker
- uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+ uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
  id: buildx
  - name: Build FVT Docker Image
  uses: docker/bake-action@33a1cc61dab930ceab0181de8cdc34a8cc55e61d # master
@@ -50,8 +50,8 @@ jobs:
  go-version: ${{ inputs.go-version }}
  - name: Setup Docker Compose
  run: |
- curl -sSL "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose
- sudo install -m755 /tmp/docker-compose "$(dirname $(which docker-compose))"
+ curl --fail -sSL "https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose
+ sudo install -m755 /tmp/docker-compose "$(dirname $(which docker))"
  docker version --format 'Docker Engine version v{{.Server.Version}}'
  docker-compose version
  - name: Test (Functional)
@@ -66,7 +66,7 @@ jobs:
  if [ -f "fvt-kafka-${{ inputs.kafka-version }}.pcap" ]; then sudo chmod a+r "fvt-kafka-${{ inputs.kafka-version }}.pcap"; fi
  - name: Upload pcap file
  if: always()
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+ uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
  with:
  name: fvt-kafka-${{ inputs.kafka-version }}.pcap
  path: fvt-kafka-${{ inputs.kafka-version }}.pcap

diff --git a/.github/workflows/i386.yml b/.github/workflows/i386.yml
@@ -22,13 +22,13 @@ jobs:
  pull-requests: read # for golangci/golangci-lint-action to fetch pull requests
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
  persist-credentials: false
  - name: Setup Go
  uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
  with:
- go-version: 1.22.x
+ go-version: 1.23.x
  - name: staticcheck
  env:
  GOARCH: 386

diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
@@ -32,7 +32,7 @@ jobs:
 
  steps:
  - name: "Checkout code"
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+ uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
  with:
  persist-credentials: false
 
@@ -59,14 +59,14 @@ jobs:
  # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
  # format to the repository Actions tab.
  - name: "Upload artifact"
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+ uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
  with:
  name: SARIF file
  path: results.sarif
  retention-days: 5
 
  # Upload the results to GitHub's code scanning dashboard.
  - name: "Upload to code-scanning"
- uses: github/codeql-action/upload-sarif@8f596b4ae3cb3c588a5c46780b86dd53fef16c52 # v3.25.2
+ uses: github/codeql-action/upload-sarif@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0
  with:
  sarif_file: results.sarif
diff --git a/.golangci.yml b/.golangci.yml
@@ -1,4 +1,5 @@
 run:
+ go: "1.20"
  timeout: 5m
  deadline: 10m
 
@@ -57,7 +58,7 @@ linters:
  enable:
  - bodyclose
  - depguard
- - exportloopref
+ # - copyloopvar
  - dogsled
  - errcheck
  - errorlint
@@ -79,6 +80,7 @@ linters:
 
 issues:
  exclude:
+ - "G115: integer overflow conversion"
  - "G404: Use of weak random number generator"
  exclude-rules:
  # exclude some linters from running on certains files.

diff --git a/Dockerfile.kafka b/Dockerfile.kafka
@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9@sha256:f30dbf77b075215f6c827c269c073b5e0973e5cea8dacdf7ecb6a19c868f37f2
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10@sha256:5c6982aca8a8fdbc3ce304bc46e28c6a30572d6791ebd13968da197e75e4f042
 
 USER root
 
@@ -19,7 +19,7 @@ RUN cd /etc/java/java-11-openjdk/*/conf/security \
 ARG SCALA_VERSION="2.13"
 ARG KAFKA_VERSION="3.6.0"
 
-# https://github.com/apache/kafka/blob/9989b68d0d38c8f1357f78bf9d53a58c1476188d/tests/docker/Dockerfile#L46-L72
+# https://github.com/apache/kafka/blob/0eaaff88cf68bc2c24d4874ff9bc1cc2b493c24b/tests/docker/Dockerfile#L75-L103
 ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages"
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN mkdir -p "/opt/kafka-${KAFKA_VERSION}" \

diff --git a/async_producer.go b/async_producer.go
@@ -1101,7 +1101,7 @@ func (bp *brokerProducer) handleSuccess(sent *produceSet, response *ProduceRespo
  bp.parent.returnSuccesses(pSet.msgs)
  // Retriable errors
  case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition,
- ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend:
+ ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend, ErrKafkaStorageError:
  if bp.parent.conf.Producer.Retry.Max <= 0 {
  bp.parent.abandonBrokerConnection(bp.broker)
  bp.parent.returnErrors(pSet.msgs, block.Err)
@@ -1134,7 +1134,7 @@ func (bp *brokerProducer) handleSuccess(sent *produceSet, response *ProduceRespo
 
  switch block.Err {
  case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition,
- ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend:
+ ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend, ErrKafkaStorageError:
  Logger.Printf("producer/broker/%d state change to [retrying] on %s/%d because %v\n",
  bp.broker.ID(), topic, partition, block.Err)
  if bp.currentRetries[topic] == nil {