Add support for using Oxia as the metadata store for Pulsar and BookKeeper

.ci/clusters/values-oxia.yaml

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+components:
+  zookeeper: false
+  oxia: true
+  # disable functions for oxia tests since there's no support for Oxia in 
+  # BookKeeperPackagesStorage which requires Zookeeper
+  functions: false
+
+oxia:
+  initialShardCount: 1
+  replicationFactor: 1
+  server:
+    replicas: 1
+    memoryLimit: 256Mi
+    dbCacheSizeMb: 128
+    storageSize: 1Gi

.ci/values-common.yaml

@@ -86,3 +86,11 @@ proxy:
 
 toolset:
   useProxy: false
+
+oxia:
+  coordinator:
+    podMonitor:
+      enabled: false
+  server:
+    podMonitor:
+      enabled: false

.github/workflows/pulsar-helm-chart-ci.yaml

@@ -140,15 +140,22 @@ jobs:
           helm dependency build charts/pulsar
           validate_helm_template_with_k8s_version() {
             local kube_version=$1
-            echo "Validating helm template with kubeconform for k8s version $kube_version"
-            helm template charts/pulsar --set kube-prometheus-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version | \
+            shift
+            echo -n "Validating helm template with kubeconform for k8s version $kube_version"
+            if [ $# -gt 0 ]; then
+              echo " Extra args: $*"
+            else
+              echo ""
+            fi
+            helm template charts/pulsar --set kube-prometheus-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version "$@" | \
               kubeconform -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary
           }
           set -o pipefail
           for k8s_version_part in {23..30}; do
             k8s_version="1.${k8s_version_part}.0"
-            echo "Validating helm template with kubeconform for k8s version $k8s_version"
             validate_helm_template_with_k8s_version $k8s_version
+            echo "Validating with Oxia enabled"
+            validate_helm_template_with_k8s_version $k8s_version --set components.zookeeper=false --set components.oxia=true
           done
       - name: Wait for ssh connection when build fails
         # ssh access is enabled for builds in own forks
@@ -208,6 +215,9 @@ jobs:
           - name: Pulsar Manager
             values_file: .ci/clusters/values-pulsar-manager.yaml
             shortname: pulsar-manager
+          - name: Oxia
+            values_file: .ci/clusters/values-oxia.yaml
+            shortname: oxia
         include:
           - k8sVersion:
               version: "1.23.17"

charts/pulsar/templates/_autorecovery.tpl

@@ -92,6 +92,7 @@ Define autorecovery init container : verify cluster id
 */}}
 {{- define "pulsar.autorecovery.init.verify_cluster_id" -}}
 bin/apply-config-from-env.py conf/bookkeeper.conf;
+export BOOKIE_MEM="-Xmx128M";
 {{- include "pulsar.autorecovery.zookeeper.tls.settings" . -}}
 until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;

charts/pulsar/templates/_bookkeeper.tpl

@@ -92,8 +92,12 @@ Define bookie tls certs volumes
 Define bookie common config
 */}}
 {{- define "pulsar.bookkeeper.config.common" -}}
+{{- if .Values.components.zookeeper }}
 zkServers: "{{ template "pulsar.zookeeper.connect" . }}"
 zkLedgersRootPath: "{{ .Values.metadataPrefix }}/ledgers"
+{{- else if .Values.components.oxia }}
+metadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
+{{- end }}
 # enable bookkeeper http server
 httpServerEnabled: "true"
 httpServerPort: "{{ .Values.bookkeeper.ports.http }}"
@@ -123,6 +127,7 @@ Define bookie init container : verify cluster id
 {{- define "pulsar.bookkeeper.init.verify_cluster_id" -}}
 {{- if not (and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence) }}
 bin/apply-config-from-env.py conf/bookkeeper.conf;
+export BOOKIE_MEM="-Xmx128M";
 {{- include "pulsar.bookkeeper.zookeeper.tls.settings" . -}}
 until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;
@@ -132,6 +137,7 @@ bin/bookkeeper shell bookieformat -nonInteractive -force -deleteCookie || true
 {{- if and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence }}
 set -e;
 bin/apply-config-from-env.py conf/bookkeeper.conf;
+export BOOKIE_MEM="-Xmx128M";
 {{- include "pulsar.bookkeeper.zookeeper.tls.settings" . -}}
 until timeout 15 bin/bookkeeper shell whatisinstanceid; do
   sleep 3;

charts/pulsar/templates/_helpers.tpl

@@ -135,3 +135,4 @@ Lookup pull policy, default to defaultPullPolicy
 {{- define "pulsar.imagePullPolicy" -}}
 {{- printf "%s" (.image.pullPolicy | default .root.Values.defaultPullPolicy) -}}
 {{- end -}}
+

charts/pulsar/templates/_oxia.tpl

@@ -0,0 +1,110 @@
+{{/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/}}
+
+{{/*
+Probe
+*/}}
+{{- define "oxia-cluster.probe" -}}
+exec:
+  command: ["oxia", "health", "--port={{ . }}"]
+initialDelaySeconds: 10
+timeoutSeconds: 10
+{{- end }}
+
+
+{{/*
+Probe
+*/}}
+{{- define "oxia-cluster.readiness-probe" -}}
+exec:
+  command: ["oxia", "health", "--port={{ . }}", "--service=oxia-readiness"]
+initialDelaySeconds: 10
+timeoutSeconds: 10
+{{- end }}
+
+{{/*
+Probe
+*/}}
+{{- define "oxia-cluster.startup-probe" -}}
+exec:
+  command: ["oxia", "health", "--port={{ . }}"]
+initialDelaySeconds: 60
+timeoutSeconds: 10
+{{- end }}
+
+{{/*
+Define the pulsar oxia
+*/}}
+{{- define "pulsar.oxia.server.service" -}}
+{{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc
+{{- end }}
+
+{{/*
+oxia url for broker metadata
+*/}}
+{{- define "pulsar.oxia.metadata.url.broker" -}}
+{{- if .Values.components.oxia -}}
+oxia://{{ template "pulsar.oxia.server.service" . }}:{{ .Values.oxia.server.ports.public }}/broker
+{{- end -}}
+{{- end -}}
+
+{{/*
+oxia url for bookkeeper metadata
+*/}}
+{{- define "pulsar.oxia.metadata.url.bookkeeper" -}}
+{{- if .Values.components.oxia -}}
+metadata-store:oxia://{{ template "pulsar.oxia.server.service" . }}:{{ .Values.oxia.server.ports.public }}/bookkeeper
+{{- end -}}
+{{- end -}}
+
+{{/*
+Define coordinator configmap
+*/}}
+{{- define "oxia.coordinator.config.yaml" -}}
+namespaces:
+  - name: default
+    initialShardCount: {{ .Values.oxia.initialShardCount }}
+    replicationFactor: {{ .Values.oxia.replicationFactor }}
+  - name: broker
+    initialShardCount: {{ .Values.oxia.initialShardCount }}
+    replicationFactor: {{ .Values.oxia.replicationFactor }}
+  - name: bookkeeper
+    initialShardCount: {{ .Values.oxia.initialShardCount }}
+    replicationFactor: {{ .Values.oxia.replicationFactor }}
+servers:
+  - public:  {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc.{{ template "pulsar.namespace" . }}.svc.cluster.local:{{ .Values.oxia.server.ports.public }}
+    internal:  {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc.{{ template "pulsar.namespace" . }}.svc:{{ .Values.oxia.server.ports.internal }}
+{{- end }}
+
+{{/*
+Define coordinator entrypoint
+*/}}
+{{- define "oxia.coordinator.entrypoint" -}}
+- "oxia"
+- "coordinator"
+- "--conf=configmap:{{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator"
+- "--log-json"
+- "--metadata=configmap"
+- "--k8s-namespace={{ template "pulsar.namespace" . }}"
+- "--k8s-configmap-name={{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator-status"
+{{- if .Values.oxia.pprofEnabled }}
+- "--profile"
+{{- end}}
+{{- end}}
+

charts/pulsar/templates/_zookeeper.tpl

@@ -56,3 +56,4 @@ Define zookeeper tls settings
 /pulsar/keytool/keytool.sh zookeeper {{ template "pulsar.zookeeper.hostname" . }} false;
 {{- end }}
 {{- end }}
+

charts/pulsar/templates/autorecovery-service-account.yaml

@@ -26,8 +26,8 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.autorecovery.component }}
-  annotations:
 {{- with .Values.autorecovery.service_account.annotations }}
+  annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
 {{- end }}

charts/pulsar/templates/autorecovery-statefulset.yaml

@@ -110,7 +110,7 @@ spec:
       terminationGracePeriodSeconds: {{ .Values.autorecovery.gracePeriod }}
       serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
       initContainers:
-      {{- if and .Values.autorecovery.waitBookkeeperTimeout (not (eq (.Values.autorecovery.waitBookkeeperTimeout | toString) "0")) }}
+      {{- if and .Values.autorecovery.waitBookkeeperTimeout (gt (.Values.autorecovery.waitBookkeeperTimeout | int) 0) }}
       # This initContainer will wait for bookkeeper initnewcluster to complete
       # before deploying the bookies
       - name: pulsar-bookkeeper-verify-clusterid

charts/pulsar/templates/bookkeeper-cluster-initialize.yaml

@@ -36,16 +36,16 @@ spec:
     spec:
     {{- include "pulsar.imagePullSecrets" . | nindent 6 }}
       serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
+      {{- with .Values.pulsar_metadata.nodeSelector }}
       nodeSelector:
-      {{- if .Values.pulsar_metadata.nodeSelector }}
 {{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }}
       {{- end }}
+      {{- with .Values.pulsar_metadata.tolerations }}
       tolerations:
-      {{- if .Values.pulsar_metadata.tolerations }}
 {{ toYaml .Values.pulsar_metadata.tolerations | indent 8 }}
       {{- end }}
       initContainers:
-      {{- if and .Values.bookkeeper.metadata.waitZookeeperTimeout (not (eq (.Values.bookkeeper.metadata.waitZookeeperTimeout | toString) "0")) }}
+      {{- if and .Values.components.zookeeper .Values.bookkeeper.metadata.waitZookeeperTimeout (gt (.Values.bookkeeper.metadata.waitZookeeperTimeout | int) 0) }}
       - name: wait-zookeeper-ready
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
         imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
@@ -64,6 +64,18 @@ spec:
             done;
             {{- end}}
       {{- end}}
+      {{- if and .Values.components.oxia .Values.bookkeeper.metadata.waitOxiaTimeout (gt (.Values.bookkeeper.metadata.waitOxiaTimeout | int) 0) }}
+      - name: wait-oxia-ready
+        image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
+        imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}"
+        resources: {{ toYaml .Values.initContainer.resources | nindent 10 }}
+        command: ["timeout", "{{ .Values.bookkeeper.metadata.waitOxiaTimeout }}", "sh", "-c"]
+        args:
+          - >-
+            until nslookup {{ template "pulsar.oxia.server.service" . }}; do
+              sleep 3;
+            done;
+      {{- end }}
       containers:
       - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-init"
         image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}"
@@ -81,7 +93,7 @@ spec:
             if timeout 15 bin/bookkeeper shell whatisinstanceid; then
                 echo "bookkeeper cluster already initialized";
             else
-                {{- if not (eq .Values.metadataPrefix "") }}
+                {{- if and .Values.components.zookeeper (not (eq .Values.metadataPrefix "")) }}
                 bin/pulsar zookeeper-shell -server {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }} create {{ .Values.metadataPrefix }} && echo 'created for pulsar cluster "{{ template "pulsar.cluster.name" . }}"' &&
                 {{- end }}
                 bin/bookkeeper shell initnewcluster;

charts/pulsar/templates/bookkeeper-service-account.yaml

@@ -26,8 +26,8 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.bookkeeper.component }}
-  annotations:
 {{- with .Values.bookkeeper.service_account.annotations }}
+  annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
 {{- end }}

charts/pulsar/templates/bookkeeper-statefulset.yaml

@@ -66,7 +66,7 @@ spec:
       affinity:
         {{- if and .Values.affinity.anti_affinity .Values.bookkeeper.affinity.anti_affinity}}
         podAntiAffinity:
-          {{ if eq .Values.bookkeeper.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
+          {{- if eq .Values.bookkeeper.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}}
           {{ .Values.bookkeeper.affinity.type }}:
           - labelSelector:
               matchExpressions:
@@ -83,7 +83,7 @@ spec:
                 values:
                 - {{ .Values.bookkeeper.component }}
             topologyKey: {{ .Values.bookkeeper.affinity.anti_affinity_topology_key }}
-        {{ else }}
+        {{- else }}
           {{ .Values.bookkeeper.affinity.type }}:
             - weight: 100
               podAffinityTerm:
@@ -110,7 +110,7 @@ spec:
       securityContext:
 {{ toYaml .Values.bookkeeper.securityContext | indent 8 }}
       {{- end }}
-      {{- if and .Values.bookkeeper.waitMetadataTimeout (not (eq (.Values.bookkeeper.waitMetadataTimeout | toString) "0")) }}
+      {{- if and .Values.bookkeeper.waitMetadataTimeout (gt (.Values.bookkeeper.waitMetadataTimeout | int) 0) }}
       initContainers:
       # This initContainer will wait for bookkeeper initnewcluster to complete
       # before deploying the bookies

charts/pulsar/templates/broker-configmap.yaml

@@ -28,13 +28,20 @@ metadata:
     component: {{ .Values.broker.component }}
 data:
   # Metadata settings
+  {{- if .Values.components.zookeeper }}
   zookeeperServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
   {{- if .Values.pulsar_metadata.configurationStore }}
   configurationStoreServers: "{{ template "pulsar.configurationStore.connect" . }}{{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }}"
   {{- end }}
   {{- if not .Values.pulsar_metadata.configurationStore }}
   configurationStoreServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
   {{- end }}
+  {{- end }}
+  {{- if .Values.components.oxia }}
+  metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}"
+  configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}"
+  bookkeeperMetadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
+  {{- end }}
 
   # Broker settings
   clusterName: {{ template "pulsar.cluster.name" . }}

charts/pulsar/templates/broker-service-account.yaml

@@ -26,8 +26,8 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.broker.component }}
-  annotations:
 {{- with .Values.broker.service_account.annotations }}
+  annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
 ---
@@ -42,8 +42,8 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.functions.component }}
-  annotations:
 {{- with .Values.functions.service_account.annotations }}
+  annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
 ---

charts/pulsar/templates/broker-service.yaml

@@ -26,8 +26,10 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.broker.component }}
+{{- with .Values.broker.service_account.annotations }}
   annotations:
-{{ toYaml .Values.broker.service.annotations | indent 4 }}
+{{ toYaml . | indent 4 }}
+{{- end }}
 spec:
   type: ClusterIP
   ports: