feat(kafka): enable kraft

README.md

@@ -14,6 +14,78 @@ Big thanks to the maintainers of the [deprecated chart](https://github.com/helm/
 
 For now the full list of values is not documented but you can get inspired by the values.yaml specific to each directory.
 
+## Upgrading from 21.x.x Version of This Chart to 22.x.x
+
+This version introduces a significant change by dropping support for Kafka Zookeeper and transitioning to Kafka Kraft
+mode. This change requires action on your part to ensure a smooth upgrade.
+
+### Major Changes
+
+- **Kafka Upgrade**: We have upgraded from Kafka `23.0.7` to `27.1.2`. This involves moving from Zookeeper to Kraft,
+  requiring a fresh setup of Kafka.
+
+### Migration Guide
+
+1. **Backup Your Data**: Ensure all your data is backed up before starting the migration process.
+2. **Retrieve the Cluster ID from Zookeeper** by executing:
+
+    ```shell
+    kubectl exec -it <your-zookeeper-pod> -- zkCli.sh get /cluster/id
+    ```
+
+3. **Deploy at least one Kraft controller-only** in your deployment with zookeeperMigrationMode=true. The Kraft
+    controllers will migrate the data from your Kafka ZkBroker to Kraft mode.
+
+    To do this, add the following values to your Zookeeper deployment when upgrading:
+
+    ```yaml
+    controller:
+        replicaCount: 1
+        controllerOnly: true
+        zookeeperMigrationMode: true
+    broker:
+        zookeeperMigrationMode: true
+    kraft:
+        enabled: true
+        clusterId: "<your_cluster_id>"
+    ```
+
+4. **Wait until all brokers are ready.** You should see the following log in the broker logs:
+
+    ```shell
+    INFO [KafkaServer id=100] Finished catching up on KRaft metadata log, requesting that the KRaft controller unfence this broker (kafka.server.KafkaServer)
+    INFO [BrokerLifecycleManager id=100 isZkBroker=true] The broker has been unfenced. Transitioning from RECOVERY to RUNNING. (kafka.server.BrokerLifecycleManager)
+    ```
+    In the controllers, the following message should show up:
+    ```shell
+    Transitioning ZK migration state from PRE_MIGRATION to MIGRATION (org.apache.kafka.controller.FeatureControlManager)
+    ```
+
+5. **Once all brokers have been successfully migrated,** set **'broker.zookeeperMigrationMode=false'** to fully migrate them.
+    ```yaml
+    broker:
+      zookeeperMigrationMode: false
+    ```
+
+6. **To conclude the migration**, switch off migration mode on controllers and stop Zookeeper:
+
+    ```yaml
+    controller:
+        zookeeperMigrationMode: false
+    zookeeper:
+        enabled: false
+    ```
+    After the migration is complete, you should see the following message in your controllers:
+
+    ```shell
+    [2023-07-13 13:07:45,226] INFO [QuorumController id=1] Transitioning ZK migration state from MIGRATION to POST_MIGRATION (org.apache.kafka.controller.FeatureControlManager)
+    ```
+7. **(Optional)** If you would like to switch to a non-dedicated cluster, set **'controller.controllerOnly=false'**. This will cause controller-only nodes to switch to controller+broker nodes.
+
+    At this point, you could manually decommission broker-only nodes by reassigning its partitions to controller-eligible nodes.
+
+    For more information about decommissioning a Kafka broker, check the official documentation.
+
 ## Upgrading from 20.x.x version of this Chart to 21.x.x
 
 Bumped dependencies:

sentry/Chart.lock

@@ -7,7 +7,7 @@ dependencies:
   version: 17.11.3
 - name: kafka
   repository: oci://registry-1.docker.io/bitnamicharts
-  version: 23.0.7
+  version: 27.1.2
 - name: clickhouse
   repository: https://sentry-kubernetes.github.io/charts
   version: 3.7.1
@@ -23,5 +23,5 @@ dependencies:
 - name: nginx
   repository: oci://registry-1.docker.io/bitnamicharts
   version: 14.2.2
-digest: sha256:108a23a1ac0d4c2da1e133b05b8837d75da198d405d0086e01efb4e42879a36b
-generated: "2024-04-02T10:18:36.917488+03:00"
+digest: sha256:0bd083c77297940eb191d337566c47be9fe86354ea0e03887bc14cff66ad7f1b
+generated: "2024-04-07T22:25:17.6336956+02:00"

sentry/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: sentry
 description: A Helm chart for Kubernetes
 type: application
-version: 21.6.3
+version: 22.0.0
 appVersion: 24.2.0
 dependencies:
   - name: memcached
@@ -15,7 +15,7 @@ dependencies:
     condition: redis.enabled
   - name: kafka
     repository: oci://registry-1.docker.io/bitnamicharts
-    version: 23.0.7
+    version: 27.1.2
     condition: kafka.enabled
   - name: clickhouse
     repository: https://sentry-kubernetes.github.io/charts

sentry/templates/hooks/sentry-db-check.job.yaml

@@ -113,14 +113,46 @@ spec:
               KAFKA_STATUS=1
               {{- if .Values.kafka.enabled }}
               KAFKA_REPLICAS={{ .Values.kafka.replicaCount }}
-              i=0; while [ $i -lt $KAFKA_REPLICAS ]; do
-                KAFKA_HOST={{ $kafkaHost }}-$i.{{ $kafkaHost }}-headless
-                if ! nc -z "$KAFKA_HOST" {{ $kafkaPort }}; then
-                  KAFKA_STATUS=0
-                  echo "$KAFKA_HOST is not available yet"
+              {{- if .Values.kafka.zookeeper.enabled }}
+              echo "Kafka Zookeeper is enabled, checking if Zookeeper is up"
+              ZOOKEEPER_STATUS=0
+              while [ $ZOOKEEPER_STATUS -eq 0 ]; do
+                ZOOKEEPER_STATUS=1
+                i=0; while [ $i -lt $KAFKA_REPLICAS ]; do
+                  ZOOKEEPER_HOST={{ $kafkaHost }}-$i.{{ $kafkaHost }}-headless
+                  if ! nc -z "$ZOOKEEPER_HOST" {{ .Values.kafka.zookeeper.port }}; then
+                    ZOOKEEPER_STATUS=0
+                    echo "$ZOOKEEPER_HOST is not available yet"
+                  fi
+                  i=$((i+1))
+                done
+                if [ "$ZOOKEEPER_STATUS" -eq 0 ]; then
+                  echo "Zookeeper not ready. Sleeping for 10s before trying again"
+                  sleep 10;
                 fi
-                i=$((i+1))
               done
+              echo "Zookeeper is up"
+              {{- end }}
+              {{- if .Values.kafka.kraft.enabled }}
+              echo "Kafka Kraft is enabled, checking if Kraft controllers are up"
+              KRAFT_STATUS=0
+              while [ $KRAFT_STATUS -eq 0 ]; do
+                KRAFT_STATUS=1
+                i=0; while [ $i -lt $KAFKA_REPLICAS ]; do
+                  KRAFT_HOST={{ $kafkaHost }}-controller-$i.{{ $kafkaHost }}-controller-headless
+                  if ! nc -z "$KRAFT_HOST" {{ $kafkaPort }}; then
+                    KRAFT_STATUS=0
+                    echo "$KRAFT_HOST is not available yet"
+                  fi
+                  i=$((i+1))
+                done
+                if [ "$KRAFT_STATUS" -eq 0 ]; then
+                  echo "Kraft controllers not ready. Sleeping for 10s before trying again"
+                  sleep 10;
+                fi
+              done
+              echo "Kraft controllers are up"
+              {{- end }}
               {{- else if (not (kindIs "slice" .Values.externalKafka)) }}
               KAFKA_HOST={{ .Values.externalKafka.host }}
               if ! nc -z "$KAFKA_HOST" {{ $kafkaPort }}; then

sentry/values.yaml

@@ -1340,16 +1340,6 @@ zookeeper:
 # See https://github.com/bitnami/charts/tree/master/bitnami/kafka
 kafka:
   enabled: true
-  replicaCount: 3
-  allowPlaintextListener: true
-  defaultReplicationFactor: 3
-  offsetsTopicReplicationFactor: 3
-  transactionStateLogReplicationFactor: 3
-  transactionStateLogMinIsr: 3
-  # 50 MB
-  maxMessageBytes: "50000000"
-  # 50 MB
-  socketRequestMaxBytes: "50000000"
   provisioning:
     enabled: true
     # Topic list is based on files below.
@@ -1431,13 +1421,19 @@ kafka:
       - name: profiles
       - name: ingest-occurrences
       - name: snuba-spans
+  listeners:
+    client:
+      protocol: "PLAINTEXT"
+    controller:
+      protocol: "PLAINTEXT"
+    interBroker:
+      protocol: "PLAINTEXT"
+    external:
+      protocol: "PLAINTEXT"
   zookeeper:
-    enabled: true
-  kraft:
     enabled: false
-  service:
-    ports:
-      client: 9092
+  kraft:
+    enabled: true
 
   ## Use this to enable an extra service account
   # serviceAccount: