Merge branch 'apache:master' into feature/pulsar_manager_initialze

.ci/chart_test.sh

@@ -35,6 +35,8 @@ source ${PULSAR_HOME}/.ci/helm.sh
 # create cluster
 ci::create_cluster
 
+ci::helm_repo_add
+
 extra_opts=""
 if [[ "x${SYMMETRIC}" == "xtrue" ]]; then
     extra_opts="-s"

.ci/helm.sh

@@ -83,6 +83,13 @@ function ci::install_cert_manager() {
     echo "Successfully installed the cert manager."
 }
 
+function ci::helm_repo_add() {
+    echo "Adding the helm repo ..."
+    ${HELM} repo add prometheus-community https://prometheus-community.github.io/helm-charts
+    ${HELM} repo update
+    echo "Successfully added the helm repo."
+}
+
 function ci::print_pod_logs() {
     echo "Logs for all pulsar containers:"
     for k8sobject in $(${KUBECTL} get pods,jobs -n ${NAMESPACE} -l app=pulsar -o=name); do

.github/workflows/pulsar-helm-chart-ci.yaml

@@ -127,6 +127,29 @@ jobs:
             --validate-maintainers=false \
             --target-branch ${{ github.event.repository.default_branch }}
 
+      - name: Run kubeconform check for helm template with every major k8s version 1.21.0-1.29.0
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: |
+          PULSAR_CHART_HOME=$(pwd)
+          source ${PULSAR_CHART_HOME}/hack/common.sh
+          source ${PULSAR_CHART_HOME}/.ci/helm.sh
+          hack::ensure_kubectl
+          hack::ensure_helm
+          hack::ensure_kubeconform
+          ci::helm_repo_add
+          helm dependency build charts/pulsar
+          validate_helm_template_with_k8s_version() {
+            local kube_version=$1
+            echo "Validating helm template with kubeconform for k8s version $kube_version"
+            helm template charts/pulsar --set kube-prometheus-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version | \
+              kubeconform -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary
+          }
+          set -o pipefail
+          for k8s_version_part in {21..29}; do
+            k8s_version="1.${k8s_version_part}.0"
+            echo "Validating helm template with kubeconform for k8s version $k8s_version"
+            validate_helm_template_with_k8s_version $k8s_version
+          done
       - name: Wait for ssh connection when build fails
         # ssh access is enabled for builds in own forks
         uses: ./.github/actions/ssh-access

charts/pulsar/values.yaml

@@ -601,11 +601,33 @@ bookkeeper:
       -XX:-ResizePLAB
       -XX:+ExitOnOutOfMemoryError
       -XX:+PerfDisableSharedMem
-    # configure the memory settings based on jvm memory settings
-    dbStorage_writeCacheMaxSizeMb: "32"
-    dbStorage_readAheadCacheMaxSizeMb: "32"
-    dbStorage_rocksDB_writeBufferSizeMB: "8"
-    dbStorage_rocksDB_blockCacheSize: "8388608"
+    #
+    # Bookkeeper configuration reference: https://bookkeeper.apache.org/docs/reference/config
+    #
+    # https://bookkeeper.apache.org/docs/reference/config#db-ledger-storage-settings
+    # You could use the below example settings for a minimal configuration
+    # dbStorage_writeCacheMaxSizeMb: "32"
+    # dbStorage_readAheadCacheMaxSizeMb: "32"
+    # dbStorage_rocksDB_writeBufferSizeMB: "8"
+    # dbStorage_rocksDB_blockCacheSize: "8388608"
+    #
+    # configure the data compaction (bookie entry log compaction and gc) settings
+    # https://bookkeeper.apache.org/docs/reference/config#garbage-collection-settings
+    # https://bookkeeper.apache.org/docs/reference/config#entry-log-compaction-settings
+    minorCompactionThreshold: "0.2"     # default 0.2 (use default)
+    minorCompactionInterval: "360"      # default 3600 seconds (6 minutes vs default 1 hour)
+    majorCompactionThreshold: "0.8"     # default 0.5
+    majorCompactionInterval: "10800"    # default 86400 seconds (3 hours vs default 1 day)
+    gcWaitTime: "300000"                # default 900000 milli-seconds (5 minutes vs default 15 minutes)
+    isForceGCAllowWhenNoSpace: "true"   # default false
+    # disk utilization configuration
+    # https://bookkeeper.apache.org/docs/reference/config#disk-utilization
+    # Make sure that diskUsageLwmThreshold <= diskUsageWarnThreshold <= diskUsageThreshold
+    diskUsageLwmThreshold: "0.85"   # default 0.90
+    diskUsageWarnThreshold: "0.9"   # default 0.95
+    diskUsageThreshold: "0.95"      # default 0.95 (use default)
+    diskCheckInterval: "1800"       # default 10000
+
   ## Add a custom command to the start up process of the bookie pods (e.g. update-ca-certificates, jvm commands, etc)
   additionalCommand:
   ## Bookkeeper Service

examples/values-minikube.yaml

@@ -35,6 +35,13 @@ zookeeper:
 
 bookkeeper:
   replicaCount: 1
+  configData:
+    # minimal memory use for bookkeeper
+    # https://bookkeeper.apache.org/docs/reference/config#db-ledger-storage-settings
+    dbStorage_writeCacheMaxSizeMb: "32"
+    dbStorage_readAheadCacheMaxSizeMb: "32"
+    dbStorage_rocksDB_writeBufferSizeMB: "8"
+    dbStorage_rocksDB_blockCacheSize: "8388608"
 
 broker:
   replicaCount: 1

hack/common.sh

@@ -33,6 +33,8 @@ HELM_BIN=$OUTPUT_BIN/helm
 KIND_BIN=$OUTPUT_BIN/kind
 CR_BIN=$OUTPUT_BIN/cr
 : "${CR_VERSION:=1.6.0}"
+KUBECONFORM_BIN=$OUTPUT_BIN/kubeconform
+: "${KUBECONFORM_VERSION:=0.6.4}"
 export PATH="$OUTPUT_BIN:$PATH"
 
 test -d "$OUTPUT_BIN" || mkdir -p "$OUTPUT_BIN"
@@ -134,3 +136,9 @@ function hack::ensure_cr() {
     chmod +x $CR_BIN
     $CR_BIN version
 }
+
+function hack::ensure_kubeconform() {
+    echo "Installing kubeconform v$KUBECONFORM_VERSION..."
+    curl -s --retry 10 -L https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-${OS}-${ARCH}.tar.gz | tar -xzO kubeconform > $KUBECONFORM_BIN
+    chmod +x $KUBECONFORM_BIN
+}