diff --git a/.ci/clusters/values-oxia.yaml b/.ci/clusters/values-oxia.yaml new file mode 100644 index 00000000..51afd8c5 --- /dev/null +++ b/.ci/clusters/values-oxia.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +components: + zookeeper: false + oxia: true + # disable functions for oxia tests since there's no support for Oxia in + # BookKeeperPackagesStorage which requires Zookeeper + functions: false + +oxia: + initialShardCount: 1 + replicationFactor: 1 + server: + replicas: 1 + memoryLimit: 256Mi + dbCacheSizeMb: 128 + storageSize: 1Gi diff --git a/.ci/values-common.yaml b/.ci/values-common.yaml index 65f23248..c8b99131 100644 --- a/.ci/values-common.yaml +++ b/.ci/values-common.yaml @@ -86,3 +86,11 @@ proxy: toolset: useProxy: false + +oxia: + coordinator: + podMonitor: + enabled: false + server: + podMonitor: + enabled: false diff --git a/.github/workflows/pulsar-helm-chart-ci.yaml b/.github/workflows/pulsar-helm-chart-ci.yaml index f9fdf1b7..027647e9 100644 --- a/.github/workflows/pulsar-helm-chart-ci.yaml +++ b/.github/workflows/pulsar-helm-chart-ci.yaml @@ -140,15 +140,22 @@ jobs: helm dependency build charts/pulsar validate_helm_template_with_k8s_version() { local kube_version=$1 - echo "Validating helm template with kubeconform for k8s version $kube_version" - helm template charts/pulsar --set kube-prometheus-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version | \ + shift + echo -n "Validating helm template with kubeconform for k8s version $kube_version" + if [ $# -gt 0 ]; then + echo " Extra args: $*" + else + echo "" + fi + helm template charts/pulsar --set kube-prometheus-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version "$@" | \ kubeconform -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary } set -o pipefail for k8s_version_part in {23..30}; do k8s_version="1.${k8s_version_part}.0" - echo "Validating helm template with kubeconform for k8s version $k8s_version" validate_helm_template_with_k8s_version $k8s_version + echo "Validating with Oxia enabled" + validate_helm_template_with_k8s_version $k8s_version --set components.zookeeper=false --set components.oxia=true done - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks @@ -208,6 +215,9 @@ jobs: - name: Pulsar Manager values_file: .ci/clusters/values-pulsar-manager.yaml shortname: pulsar-manager + - name: Oxia + values_file: .ci/clusters/values-oxia.yaml + shortname: oxia include: - k8sVersion: version: "1.23.17" diff --git a/charts/pulsar/templates/_autorecovery.tpl b/charts/pulsar/templates/_autorecovery.tpl index 1279fbca..8ba78c84 100644 --- a/charts/pulsar/templates/_autorecovery.tpl +++ b/charts/pulsar/templates/_autorecovery.tpl @@ -92,6 +92,7 @@ Define autorecovery init container : verify cluster id */}} {{- define "pulsar.autorecovery.init.verify_cluster_id" -}} bin/apply-config-from-env.py conf/bookkeeper.conf; +export BOOKIE_MEM="-Xmx128M"; {{- include "pulsar.autorecovery.zookeeper.tls.settings" . -}} until timeout 15 bin/bookkeeper shell whatisinstanceid; do sleep 3; diff --git a/charts/pulsar/templates/_bookkeeper.tpl b/charts/pulsar/templates/_bookkeeper.tpl index 6cef3437..d7a57029 100644 --- a/charts/pulsar/templates/_bookkeeper.tpl +++ b/charts/pulsar/templates/_bookkeeper.tpl @@ -92,8 +92,12 @@ Define bookie tls certs volumes Define bookie common config */}} {{- define "pulsar.bookkeeper.config.common" -}} +{{- if .Values.components.zookeeper }} zkServers: "{{ template "pulsar.zookeeper.connect" . }}" zkLedgersRootPath: "{{ .Values.metadataPrefix }}/ledgers" +{{- else if .Values.components.oxia }} +metadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}" +{{- end }} # enable bookkeeper http server httpServerEnabled: "true" httpServerPort: "{{ .Values.bookkeeper.ports.http }}" @@ -123,6 +127,7 @@ Define bookie init container : verify cluster id {{- define "pulsar.bookkeeper.init.verify_cluster_id" -}} {{- if not (and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence) }} bin/apply-config-from-env.py conf/bookkeeper.conf; +export BOOKIE_MEM="-Xmx128M"; {{- include "pulsar.bookkeeper.zookeeper.tls.settings" . -}} until timeout 15 bin/bookkeeper shell whatisinstanceid; do sleep 3; @@ -132,6 +137,7 @@ bin/bookkeeper shell bookieformat -nonInteractive -force -deleteCookie || true {{- if and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence }} set -e; bin/apply-config-from-env.py conf/bookkeeper.conf; +export BOOKIE_MEM="-Xmx128M"; {{- include "pulsar.bookkeeper.zookeeper.tls.settings" . -}} until timeout 15 bin/bookkeeper shell whatisinstanceid; do sleep 3; diff --git a/charts/pulsar/templates/_helpers.tpl b/charts/pulsar/templates/_helpers.tpl index 62eb0466..d8bc866d 100644 --- a/charts/pulsar/templates/_helpers.tpl +++ b/charts/pulsar/templates/_helpers.tpl @@ -135,3 +135,4 @@ Lookup pull policy, default to defaultPullPolicy {{- define "pulsar.imagePullPolicy" -}} {{- printf "%s" (.image.pullPolicy | default .root.Values.defaultPullPolicy) -}} {{- end -}} + diff --git a/charts/pulsar/templates/_oxia.tpl b/charts/pulsar/templates/_oxia.tpl new file mode 100644 index 00000000..95afc5f7 --- /dev/null +++ b/charts/pulsar/templates/_oxia.tpl @@ -0,0 +1,110 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{/* +Probe +*/}} +{{- define "oxia-cluster.probe" -}} +exec: + command: ["oxia", "health", "--port={{ . }}"] +initialDelaySeconds: 10 +timeoutSeconds: 10 +{{- end }} + + +{{/* +Probe +*/}} +{{- define "oxia-cluster.readiness-probe" -}} +exec: + command: ["oxia", "health", "--port={{ . }}", "--service=oxia-readiness"] +initialDelaySeconds: 10 +timeoutSeconds: 10 +{{- end }} + +{{/* +Probe +*/}} +{{- define "oxia-cluster.startup-probe" -}} +exec: + command: ["oxia", "health", "--port={{ . }}"] +initialDelaySeconds: 60 +timeoutSeconds: 10 +{{- end }} + +{{/* +Define the pulsar oxia +*/}} +{{- define "pulsar.oxia.server.service" -}} +{{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc +{{- end }} + +{{/* +oxia url for broker metadata +*/}} +{{- define "pulsar.oxia.metadata.url.broker" -}} +{{- if .Values.components.oxia -}} +oxia://{{ template "pulsar.oxia.server.service" . }}:{{ .Values.oxia.server.ports.public }}/broker +{{- end -}} +{{- end -}} + +{{/* +oxia url for bookkeeper metadata +*/}} +{{- define "pulsar.oxia.metadata.url.bookkeeper" -}} +{{- if .Values.components.oxia -}} +metadata-store:oxia://{{ template "pulsar.oxia.server.service" . }}:{{ .Values.oxia.server.ports.public }}/bookkeeper +{{- end -}} +{{- end -}} + +{{/* +Define coordinator configmap +*/}} +{{- define "oxia.coordinator.config.yaml" -}} +namespaces: + - name: default + initialShardCount: {{ .Values.oxia.initialShardCount }} + replicationFactor: {{ .Values.oxia.replicationFactor }} + - name: broker + initialShardCount: {{ .Values.oxia.initialShardCount }} + replicationFactor: {{ .Values.oxia.replicationFactor }} + - name: bookkeeper + initialShardCount: {{ .Values.oxia.initialShardCount }} + replicationFactor: {{ .Values.oxia.replicationFactor }} +servers: + - public: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc.{{ template "pulsar.namespace" . }}.svc.cluster.local:{{ .Values.oxia.server.ports.public }} + internal: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc.{{ template "pulsar.namespace" . }}.svc:{{ .Values.oxia.server.ports.internal }} +{{- end }} + +{{/* +Define coordinator entrypoint +*/}} +{{- define "oxia.coordinator.entrypoint" -}} +- "oxia" +- "coordinator" +- "--conf=configmap:{{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator" +- "--log-json" +- "--metadata=configmap" +- "--k8s-namespace={{ template "pulsar.namespace" . }}" +- "--k8s-configmap-name={{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator-status" +{{- if .Values.oxia.pprofEnabled }} +- "--profile" +{{- end}} +{{- end}} + diff --git a/charts/pulsar/templates/_zookeeper.tpl b/charts/pulsar/templates/_zookeeper.tpl index 08f7426c..215b4552 100644 --- a/charts/pulsar/templates/_zookeeper.tpl +++ b/charts/pulsar/templates/_zookeeper.tpl @@ -56,3 +56,4 @@ Define zookeeper tls settings /pulsar/keytool/keytool.sh zookeeper {{ template "pulsar.zookeeper.hostname" . }} false; {{- end }} {{- end }} + diff --git a/charts/pulsar/templates/autorecovery-service-account.yaml b/charts/pulsar/templates/autorecovery-service-account.yaml index bbb66bb0..e72580b9 100644 --- a/charts/pulsar/templates/autorecovery-service-account.yaml +++ b/charts/pulsar/templates/autorecovery-service-account.yaml @@ -26,8 +26,8 @@ metadata: labels: {{- include "pulsar.standardLabels" . | nindent 4 }} component: {{ .Values.autorecovery.component }} - annotations: {{- with .Values.autorecovery.service_account.annotations }} + annotations: {{ toYaml . | indent 4 }} {{- end }} {{- end }} diff --git a/charts/pulsar/templates/autorecovery-statefulset.yaml b/charts/pulsar/templates/autorecovery-statefulset.yaml index 81327e16..811b2404 100644 --- a/charts/pulsar/templates/autorecovery-statefulset.yaml +++ b/charts/pulsar/templates/autorecovery-statefulset.yaml @@ -110,7 +110,7 @@ spec: terminationGracePeriodSeconds: {{ .Values.autorecovery.gracePeriod }} serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}" initContainers: - {{- if and .Values.autorecovery.waitBookkeeperTimeout (not (eq (.Values.autorecovery.waitBookkeeperTimeout | toString) "0")) }} + {{- if and .Values.autorecovery.waitBookkeeperTimeout (gt (.Values.autorecovery.waitBookkeeperTimeout | int) 0) }} # This initContainer will wait for bookkeeper initnewcluster to complete # before deploying the bookies - name: pulsar-bookkeeper-verify-clusterid diff --git a/charts/pulsar/templates/bookkeeper-cluster-initialize.yaml b/charts/pulsar/templates/bookkeeper-cluster-initialize.yaml index d76f25a3..bfee3a6b 100755 --- a/charts/pulsar/templates/bookkeeper-cluster-initialize.yaml +++ b/charts/pulsar/templates/bookkeeper-cluster-initialize.yaml @@ -36,16 +36,16 @@ spec: spec: {{- include "pulsar.imagePullSecrets" . | nindent 6 }} serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}" + {{- with .Values.pulsar_metadata.nodeSelector }} nodeSelector: - {{- if .Values.pulsar_metadata.nodeSelector }} {{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }} {{- end }} + {{- with .Values.pulsar_metadata.tolerations }} tolerations: - {{- if .Values.pulsar_metadata.tolerations }} {{ toYaml .Values.pulsar_metadata.tolerations | indent 8 }} {{- end }} initContainers: - {{- if and .Values.bookkeeper.metadata.waitZookeeperTimeout (not (eq (.Values.bookkeeper.metadata.waitZookeeperTimeout | toString) "0")) }} + {{- if and .Values.components.zookeeper .Values.bookkeeper.metadata.waitZookeeperTimeout (gt (.Values.bookkeeper.metadata.waitZookeeperTimeout | int) 0) }} - name: wait-zookeeper-ready image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}" imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}" @@ -64,6 +64,18 @@ spec: done; {{- end}} {{- end}} + {{- if and .Values.components.oxia .Values.bookkeeper.metadata.waitOxiaTimeout (gt (.Values.bookkeeper.metadata.waitOxiaTimeout | int) 0) }} + - name: wait-oxia-ready + image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}" + imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.bookie "root" .) }}" + resources: {{ toYaml .Values.initContainer.resources | nindent 10 }} + command: ["timeout", "{{ .Values.bookkeeper.metadata.waitOxiaTimeout }}", "sh", "-c"] + args: + - >- + until nslookup {{ template "pulsar.oxia.server.service" . }}; do + sleep 3; + done; + {{- end }} containers: - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-init" image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.bookie "root" .) }}" @@ -81,7 +93,7 @@ spec: if timeout 15 bin/bookkeeper shell whatisinstanceid; then echo "bookkeeper cluster already initialized"; else - {{- if not (eq .Values.metadataPrefix "") }} + {{- if and .Values.components.zookeeper (not (eq .Values.metadataPrefix "")) }} bin/pulsar zookeeper-shell -server {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }} create {{ .Values.metadataPrefix }} && echo 'created for pulsar cluster "{{ template "pulsar.cluster.name" . }}"' && {{- end }} bin/bookkeeper shell initnewcluster; diff --git a/charts/pulsar/templates/bookkeeper-service-account.yaml b/charts/pulsar/templates/bookkeeper-service-account.yaml index 245284e5..5779fba3 100644 --- a/charts/pulsar/templates/bookkeeper-service-account.yaml +++ b/charts/pulsar/templates/bookkeeper-service-account.yaml @@ -26,8 +26,8 @@ metadata: labels: {{- include "pulsar.standardLabels" . | nindent 4 }} component: {{ .Values.bookkeeper.component }} - annotations: {{- with .Values.bookkeeper.service_account.annotations }} + annotations: {{ toYaml . | indent 4 }} {{- end }} {{- end }} diff --git a/charts/pulsar/templates/bookkeeper-statefulset.yaml b/charts/pulsar/templates/bookkeeper-statefulset.yaml index 4c0e5099..16799bc1 100644 --- a/charts/pulsar/templates/bookkeeper-statefulset.yaml +++ b/charts/pulsar/templates/bookkeeper-statefulset.yaml @@ -66,7 +66,7 @@ spec: affinity: {{- if and .Values.affinity.anti_affinity .Values.bookkeeper.affinity.anti_affinity}} podAntiAffinity: - {{ if eq .Values.bookkeeper.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}} + {{- if eq .Values.bookkeeper.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}} {{ .Values.bookkeeper.affinity.type }}: - labelSelector: matchExpressions: @@ -83,7 +83,7 @@ spec: values: - {{ .Values.bookkeeper.component }} topologyKey: {{ .Values.bookkeeper.affinity.anti_affinity_topology_key }} - {{ else }} + {{- else }} {{ .Values.bookkeeper.affinity.type }}: - weight: 100 podAffinityTerm: @@ -110,7 +110,7 @@ spec: securityContext: {{ toYaml .Values.bookkeeper.securityContext | indent 8 }} {{- end }} - {{- if and .Values.bookkeeper.waitMetadataTimeout (not (eq (.Values.bookkeeper.waitMetadataTimeout | toString) "0")) }} + {{- if and .Values.bookkeeper.waitMetadataTimeout (gt (.Values.bookkeeper.waitMetadataTimeout | int) 0) }} initContainers: # This initContainer will wait for bookkeeper initnewcluster to complete # before deploying the bookies diff --git a/charts/pulsar/templates/broker-configmap.yaml b/charts/pulsar/templates/broker-configmap.yaml index 1f348752..60fbf96f 100644 --- a/charts/pulsar/templates/broker-configmap.yaml +++ b/charts/pulsar/templates/broker-configmap.yaml @@ -28,6 +28,7 @@ metadata: component: {{ .Values.broker.component }} data: # Metadata settings + {{- if .Values.components.zookeeper }} zookeeperServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}" {{- if .Values.pulsar_metadata.configurationStore }} configurationStoreServers: "{{ template "pulsar.configurationStore.connect" . }}{{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }}" @@ -35,6 +36,12 @@ data: {{- if not .Values.pulsar_metadata.configurationStore }} configurationStoreServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}" {{- end }} + {{- end }} + {{- if .Values.components.oxia }} + metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}" + configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url.broker" . }}" + bookkeeperMetadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}" + {{- end }} # Broker settings clusterName: {{ template "pulsar.cluster.name" . }} diff --git a/charts/pulsar/templates/broker-service-account.yaml b/charts/pulsar/templates/broker-service-account.yaml index a63c67bf..5e239762 100644 --- a/charts/pulsar/templates/broker-service-account.yaml +++ b/charts/pulsar/templates/broker-service-account.yaml @@ -26,8 +26,8 @@ metadata: labels: {{- include "pulsar.standardLabels" . | nindent 4 }} component: {{ .Values.broker.component }} - annotations: {{- with .Values.broker.service_account.annotations }} + annotations: {{ toYaml . | indent 4 }} {{- end }} --- @@ -42,8 +42,8 @@ metadata: labels: {{- include "pulsar.standardLabels" . | nindent 4 }} component: {{ .Values.functions.component }} - annotations: {{- with .Values.functions.service_account.annotations }} + annotations: {{ toYaml . | indent 4 }} {{- end }} --- diff --git a/charts/pulsar/templates/broker-service.yaml b/charts/pulsar/templates/broker-service.yaml index f9cd1c74..0c683c4b 100644 --- a/charts/pulsar/templates/broker-service.yaml +++ b/charts/pulsar/templates/broker-service.yaml @@ -26,8 +26,10 @@ metadata: labels: {{- include "pulsar.standardLabels" . | nindent 4 }} component: {{ .Values.broker.component }} +{{- with .Values.broker.service_account.annotations }} annotations: -{{ toYaml .Values.broker.service.annotations | indent 4 }} +{{ toYaml . | indent 4 }} +{{- end }} spec: type: ClusterIP ports: diff --git a/charts/pulsar/templates/broker-statefulset.yaml b/charts/pulsar/templates/broker-statefulset.yaml index c494c202..45915ffe 100644 --- a/charts/pulsar/templates/broker-statefulset.yaml +++ b/charts/pulsar/templates/broker-statefulset.yaml @@ -87,7 +87,7 @@ spec: affinity: {{- if and .Values.affinity.anti_affinity .Values.broker.affinity.anti_affinity}} podAntiAffinity: - {{ if eq .Values.broker.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}} + {{- if eq .Values.broker.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}} {{ .Values.broker.affinity.type }}: - labelSelector: matchExpressions: @@ -104,7 +104,7 @@ spec: values: - {{ .Values.broker.component }} topologyKey: {{ .Values.broker.affinity.anti_affinity_topology_key }} - {{ else }} + {{- else }} {{ .Values.broker.affinity.type }}: - weight: 100 podAffinityTerm: @@ -123,11 +123,11 @@ spec: values: - {{ .Values.broker.component }} topologyKey: {{ .Values.broker.affinity.anti_affinity_topology_key }} - {{ end }} + {{- end }} {{- end }} terminationGracePeriodSeconds: {{ .Values.broker.gracePeriod }} initContainers: - {{- if and .Values.broker.waitZookeeperTimeout (not (eq (.Values.broker.waitZookeeperTimeout | toString) "0")) }} + {{- if and .Values.components.zookeeper .Values.broker.waitZookeeperTimeout (gt (.Values.broker.waitZookeeperTimeout | int) 0) }} # This init container will wait for zookeeper to be ready before # deploying the bookies - name: wait-zookeeper-ready @@ -154,7 +154,19 @@ spec: volumeMounts: {{- include "pulsar.broker.certs.volumeMounts" . | nindent 8 }} {{- end }} - {{- if and .Values.broker.waitBookkeeperTimeout (not (eq (.Values.broker.waitBookkeeperTimeout | toString) "0")) }} + {{- if and .Values.components.oxia .Values.broker.waitOxiaTimeout (gt (.Values.broker.waitOxiaTimeout | int) 0) }} + - name: wait-oxia-ready + image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.broker "root" .) }}" + imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.broker "root" .) }}" + resources: {{ toYaml .Values.initContainer.resources | nindent 10 }} + command: ["timeout", "{{ .Values.broker.waitOxiaTimeout }}", "sh", "-c"] + args: + - >- + until nslookup {{ template "pulsar.oxia.server.service" . }}; do + sleep 3; + done; + {{- end }} + {{- if and .Values.broker.waitBookkeeperTimeout (gt (.Values.broker.waitBookkeeperTimeout | int) 0) }} # This init container will wait for bookkeeper to be ready before # deploying the broker - name: wait-bookkeeper-ready @@ -239,6 +251,7 @@ spec: bin/apply-config-from-env.py conf/broker.conf; bin/gen-yml-from-env.py conf/functions_worker.yml; echo "OK" > "${statusFilePath:-status}"; + {{- if .Values.components.zookeeper }} {{- include "pulsar.broker.zookeeper.tls.settings" . | nindent 10 }} timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }}; while [ $? -eq 0 ]; do @@ -246,6 +259,7 @@ spec: sleep 10; timeout 15 bin/pulsar zookeeper-shell -server {{ template "pulsar.zookeeper.connect" . }} get {{ template "pulsar.broker.znode" . }}; done; + {{- end }} cat conf/pulsar_env.sh; OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/pulsar broker; ports: diff --git a/charts/pulsar/templates/oxia-coordinator-configmap.yaml b/charts/pulsar/templates/oxia-coordinator-configmap.yaml new file mode 100644 index 00000000..577dac0b --- /dev/null +++ b/charts/pulsar/templates/oxia-coordinator-configmap.yaml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +{{- if .Values.components.oxia }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-coordinator +data: + config.yaml: | + {{- include "oxia.coordinator.config.yaml" . | nindent 4 }} + +{{- end }} \ No newline at end of file diff --git a/charts/pulsar/templates/oxia-coordinator-deployment.yaml b/charts/pulsar/templates/oxia-coordinator-deployment.yaml new file mode 100644 index 00000000..3739c412 --- /dev/null +++ b/charts/pulsar/templates/oxia-coordinator-deployment.yaml @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.components.oxia }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-coordinator +spec: + replicas: 1 + selector: + matchLabels: + {{- include "pulsar.matchLabels" . | nindent 6 }} + component: {{ .Values.oxia.component }}-coordinator + strategy: + type: Recreate + template: + metadata: + labels: + {{- include "pulsar.template.labels" . | nindent 8 }} + component: {{ .Values.oxia.component }}-coordinator + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.oxia.coordinator.ports.metrics }}" + spec: + {{- if .Values.oxia.server.nodeSelector }} + nodeSelector: +{{ toYaml .Values.oxia.server.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.oxia.server.tolerations }} + tolerations: +{{ toYaml .Values.oxia.server.tolerations | indent 8 }} + {{- end }} + serviceAccountName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator + containers: + - command: + {{- include "oxia.coordinator.entrypoint" . | nindent 12 }} + image: "{{ .Values.images.oxia.repository }}:{{ .Values.images.oxia.tag }}" + imagePullPolicy: {{ .Values.images.oxia.pullPolicy }} + name: coordinator + ports: + {{- range $key, $value := .Values.oxia.coordinator.ports }} + - containerPort: {{ $value | int }} + name: {{ $key }} + {{- end}} + resources: + limits: + cpu: {{ .Values.oxia.coordinator.cpuLimit }} + memory: {{ .Values.oxia.coordinator.memoryLimit }} + livenessProbe: + {{- include "oxia-cluster.probe" .Values.oxia.coordinator.ports.internal | nindent 12 }} + readinessProbe: + {{- include "oxia-cluster.probe" .Values.oxia.coordinator.ports.internal | nindent 12 }} +{{- end }} \ No newline at end of file diff --git a/charts/pulsar/templates/oxia-coordinator-podmonitor.yaml b/charts/pulsar/templates/oxia-coordinator-podmonitor.yaml new file mode 100644 index 00000000..cddcba38 --- /dev/null +++ b/charts/pulsar/templates/oxia-coordinator-podmonitor.yaml @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# deploy oxia-coordinator PodMonitor only when `$.Values.oxia.podMonitor.enabled` is true +{{- if and $.Values.components.oxia $.Values.oxia.coordinator.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ template "pulsar.fullname" . }}-oxia-coordinator + labels: + app: {{ template "pulsar.name" . }} + chart: {{ template "pulsar.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + jobLabel: oxia-coordinator + podMetricsEndpoints: + - port: metrics + path: /metrics + scheme: http + interval: {{ $.Values.oxia.coordinator.podMonitor.interval }} + scrapeTimeout: {{ $.Values.oxia.coordinator.podMonitor.scrapeTimeout }} + relabelings: + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - sourceLabels: [__meta_kubernetes_namespace] + action: replace + targetLabel: kubernetes_namespace + - sourceLabels: [__meta_kubernetes_pod_label_component] + action: replace + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_name] + action: replace + targetLabel: kubernetes_pod_name + {{- if $.Values.oxia.coordinator.podMonitor.metricRelabelings }} + metricRelabelings: {{ toYaml $.Values.oxia.coordinator.podMonitor.metricRelabelings | nindent 8 }} + {{- end }} + selector: + matchLabels: + {{- include "pulsar.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: oxia-coordinator +{{- end }} diff --git a/charts/pulsar/templates/oxia-coordinator-role.yaml b/charts/pulsar/templates/oxia-coordinator-role.yaml new file mode 100644 index 00000000..39cfbbd8 --- /dev/null +++ b/charts/pulsar/templates/oxia-coordinator-role.yaml @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.components.oxia }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-coordinator +rules: + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "*" ] +{{- end }} \ No newline at end of file diff --git a/charts/pulsar/templates/oxia-coordinator-rolebinding.yaml b/charts/pulsar/templates/oxia-coordinator-rolebinding.yaml new file mode 100644 index 00000000..354d8df1 --- /dev/null +++ b/charts/pulsar/templates/oxia-coordinator-rolebinding.yaml @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.components.oxia }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-coordinator +subjects: + - kind: ServiceAccount + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator + namespace: {{ template "pulsar.namespace" . }} +roleRef: + apiGroup: "" + kind: Role + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator +{{- end }} \ No newline at end of file diff --git a/charts/pulsar/templates/oxia-coordinator-service.yaml b/charts/pulsar/templates/oxia-coordinator-service.yaml new file mode 100644 index 00000000..3e3cc59b --- /dev/null +++ b/charts/pulsar/templates/oxia-coordinator-service.yaml @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.components.oxia }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-coordinator +{{- with .Values.oxia.coordinator.service.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + ports: + {{- range $key, $value := .Values.oxia.coordinator.ports }} + - name: {{ $key }} + port: {{ $value }} + targetPort: {{ $key }} + {{- end}} + selector: + {{- include "pulsar.matchLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-coordinator +{{- end }} \ No newline at end of file diff --git a/charts/pulsar/templates/oxia-coordinator-serviceaccount.yaml b/charts/pulsar/templates/oxia-coordinator-serviceaccount.yaml new file mode 100644 index 00000000..fbed0041 --- /dev/null +++ b/charts/pulsar/templates/oxia-coordinator-serviceaccount.yaml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +{{- if .Values.components.oxia }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-coordinator + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-coordinator +{{- with .Values.oxia.coordinator.service_account.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +{{- if .Values.images.imagePullSecrets }} +imagePullSecrets: + - name: {{ .Values.images.imagePullSecrets.secretName }} +{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/pulsar/templates/oxia-server-podmonitor.yaml b/charts/pulsar/templates/oxia-server-podmonitor.yaml new file mode 100644 index 00000000..6f2f6122 --- /dev/null +++ b/charts/pulsar/templates/oxia-server-podmonitor.yaml @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# deploy oxia-server PodMonitor only when `$.Values.oxia.podMonitor.enabled` is true +{{- if and $.Values.components.oxia $.Values.oxia.server.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ template "pulsar.fullname" . }}-oxia-server + labels: + app: {{ template "pulsar.name" . }} + chart: {{ template "pulsar.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + jobLabel: oxia-server + podMetricsEndpoints: + - port: metrics + path: /metrics + scheme: http + interval: {{ $.Values.oxia.server.podMonitor.interval }} + scrapeTimeout: {{ $.Values.oxia.server.podMonitor.scrapeTimeout }} + relabelings: + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - sourceLabels: [__meta_kubernetes_namespace] + action: replace + targetLabel: kubernetes_namespace + - sourceLabels: [__meta_kubernetes_pod_label_component] + action: replace + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_name] + action: replace + targetLabel: kubernetes_pod_name + {{- if $.Values.oxia.server.podMonitor.metricRelabelings }} + metricRelabelings: {{ toYaml $.Values.oxia.server.podMonitor.metricRelabelings | nindent 8 }} + {{- end }} + selector: + matchLabels: + {{- include "pulsar.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: oxia-server +{{- end }} diff --git a/charts/pulsar/templates/oxia-server-service-public.yaml b/charts/pulsar/templates/oxia-server-service-public.yaml new file mode 100644 index 00000000..ff328af1 --- /dev/null +++ b/charts/pulsar/templates/oxia-server-service-public.yaml @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.components.oxia }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }} + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-server +{{- with .Values.oxia.server.service.public.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + ports: + {{- range $key, $value := .Values.oxia.server.ports }} + - name: {{ $key }} + port: {{ $value }} + targetPort: {{ $key }} + {{- end}} + selector: + {{- include "pulsar.matchLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-server +{{- end}} diff --git a/charts/pulsar/templates/oxia-server-service.yaml b/charts/pulsar/templates/oxia-server-service.yaml new file mode 100644 index 00000000..2daccd6f --- /dev/null +++ b/charts/pulsar/templates/oxia-server-service.yaml @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.components.oxia }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-server +{{- with .Values.oxia.server.service.internal.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + clusterIP: None + publishNotReadyAddresses: true + ports: + {{- range $key, $value := .Values.oxia.server.ports }} + - name: {{ $key }} + port: {{ $value }} + targetPort: {{ $key }} + {{- end}} + selector: + {{- include "pulsar.matchLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-server +{{- end}} \ No newline at end of file diff --git a/charts/pulsar/templates/oxia-server-serviceaccount.yaml b/charts/pulsar/templates/oxia-server-serviceaccount.yaml new file mode 100644 index 00000000..3158a9d4 --- /dev/null +++ b/charts/pulsar/templates/oxia-server-serviceaccount.yaml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +{{- if .Values.components.oxia }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }} + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-server +{{- with .Values.oxia.server.service_account.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +{{- if .Values.images.imagePullSecrets }} +imagePullSecrets: + - name: {{ .Values.images.imagePullSecrets.secretName }} +{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/pulsar/templates/oxia-server-statefulset.yaml b/charts/pulsar/templates/oxia-server-statefulset.yaml new file mode 100644 index 00000000..d6d7f418 --- /dev/null +++ b/charts/pulsar/templates/oxia-server-statefulset.yaml @@ -0,0 +1,147 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.components.oxia }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-server + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.oxia.component }}-server +spec: + replicas: {{ .Values.oxia.server.replicas }} + selector: + matchLabels: + {{- include "pulsar.matchLabels" . | nindent 6 }} + component: {{ .Values.oxia.component }}-server + serviceName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-svc + podManagementPolicy: Parallel + template: + metadata: + labels: + {{- include "pulsar.template.labels" . | nindent 8 }} + component: {{ .Values.oxia.component }}-server + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.oxia.server.ports.metrics }}" + spec: + {{- if .Values.oxia.server.nodeSelector }} + nodeSelector: +{{ toYaml .Values.oxia.server.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.oxia.server.tolerations }} + tolerations: +{{ toYaml .Values.oxia.server.tolerations | indent 8 }} + {{- end }} + {{- if .Values.oxia.server.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml .Values.oxia.server.topologySpreadConstraints | nindent 8 }} + {{- end }} + affinity: + {{- if and .Values.affinity.anti_affinity .Values.oxia.server.affinity.anti_affinity}} + podAntiAffinity: + {{ if eq .Values.oxia.server.affinity.type "requiredDuringSchedulingIgnoredDuringExecution"}} + {{ .Values.oxia.server.affinity.type }}: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - "{{ template "pulsar.name" . }}" + - key: "release" + operator: In + values: + - {{ .Release.Name }} + - key: "component" + operator: In + values: + - {{ .Values.oxia.component }}-server + topologyKey: {{ .Values.oxia.server.affinity.anti_affinity_topology_key }} + {{ else }} + {{ .Values.oxia.server.affinity.type }}: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - "{{ template "pulsar.name" . }}" + - key: "release" + operator: In + values: + - {{ .Release.Name }} + - key: "component" + operator: In + values: + - {{ .Values.oxia.component }}-server + topologyKey: {{ .Values.oxia.server.affinity.anti_affinity_topology_key }} + {{ end }} + {{- end }} + serviceAccountName: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }} + {{- if .Values.oxia.server.securityContext }} + securityContext: +{{ toYaml .Values.oxia.server.securityContext | indent 8 }} + {{- end }} + containers: + - command: + - "oxia" + - "server" + - "--log-json" + - "--data-dir=/data/db" + - "--wal-dir=/data/wal" + - "--db-cache-size-mb={{ .Values.oxia.server.dbCacheSizeMb }}" + {{- if .Values.oxia.pprofEnabled }} + - "--profile" + {{- end}} + image: "{{ .Values.images.oxia.repository }}:{{ .Values.images.oxia.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.images.oxia.pullPolicy }} + name: server + ports: + {{- range $key, $value := .Values.oxia.server.ports }} + - containerPort: {{ $value | int }} + name: {{ $key }} + {{- end}} + resources: + limits: + cpu: {{ .Values.oxia.server.cpuLimit }} + memory: {{ .Values.oxia.server.memoryLimit }} + volumeMounts: + - name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-data + mountPath: /data + livenessProbe: + {{- include "oxia-cluster.probe" .Values.oxia.server.ports.internal | nindent 12 }} + readinessProbe: + {{- include "oxia-cluster.readiness-probe" .Values.oxia.server.ports.internal | nindent 12 }} + startupProbe: + {{- include "oxia-cluster.startup-probe" .Values.oxia.server.ports.internal | nindent 12 }} + volumeClaimTemplates: + - metadata: + name: {{ template "pulsar.fullname" . }}-{{ .Values.oxia.component }}-data + spec: + accessModes: [ "ReadWriteOnce" ] + {{- if .Values.oxia.server.storageClassName }} + storageClassName: {{ .Values.oxia.server.storageClassName }} + {{- end}} + resources: + requests: + storage: {{ .Values.oxia.server.storageSize }} +{{- end}} \ No newline at end of file diff --git a/charts/pulsar/templates/proxy-statefulset.yaml b/charts/pulsar/templates/proxy-statefulset.yaml index 740d256e..d2728d82 100644 --- a/charts/pulsar/templates/proxy-statefulset.yaml +++ b/charts/pulsar/templates/proxy-statefulset.yaml @@ -109,7 +109,7 @@ spec: terminationGracePeriodSeconds: {{ .Values.proxy.gracePeriod }} serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}" initContainers: - {{- if and .Values.proxy.waitZookeeperTimeout (not (eq (.Values.proxy.waitZookeeperTimeout | toString) "0")) }} + {{- if and .Values.components.zookeeper .Values.proxy.waitZookeeperTimeout (gt (.Values.proxy.waitZookeeperTimeout | int) 0) }} # This init container will wait for zookeeper to be ready before # deploying the bookies - name: wait-zookeeper-ready @@ -130,7 +130,19 @@ spec: done; {{- end}} {{- end}} - {{- if and .Values.proxy.waitBrokerTimeout (not (eq (.Values.proxy.waitBrokerTimeout | toString) "0")) }} + {{- if and .Values.components.oxia .Values.proxy.waitOxiaTimeout (gt (.Values.proxy.waitOxiaTimeout | int) 0) }} + - name: wait-oxia-ready + image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.proxy "root" .) }}" + imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.images.proxy "root" .) }}" + resources: {{ toYaml .Values.initContainer.resources | nindent 10 }} + command: ["timeout", "{{ .Values.proxy.waitOxiaTimeout }}", "sh", "-c"] + args: + - >- + until nslookup {{ template "pulsar.oxia.server.service" . }}; do + sleep 3; + done; + {{- end }} + {{- if and .Values.proxy.waitBrokerTimeout (gt (.Values.proxy.waitBrokerTimeout | int) 0) }} # This init container will wait for at least one broker to be ready before # deploying the proxy - name: wait-broker-ready diff --git a/charts/pulsar/templates/pulsar-cluster-initialize.yaml b/charts/pulsar/templates/pulsar-cluster-initialize.yaml index b34494b2..ad3a53d2 100755 --- a/charts/pulsar/templates/pulsar-cluster-initialize.yaml +++ b/charts/pulsar/templates/pulsar-cluster-initialize.yaml @@ -22,7 +22,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }}" + name: {{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }} namespace: {{ template "pulsar.namespace" . }} labels: {{- include "pulsar.standardLabels" . | nindent 4 }} @@ -41,9 +41,9 @@ spec: {{ toYaml .Values.pulsar_metadata.nodeSelector | indent 8 }} {{- end }} initContainers: - {{- if and .Values.pulsar_metadata.waitZookeeperTimeout (not (eq (.Values.pulsar_metadata.waitZookeeperTimeout | toString) "0")) }} + {{- if and .Values.components.zookeeper .Values.pulsar_metadata.waitZookeeperTimeout (gt (.Values.pulsar_metadata.waitZookeeperTimeout | int) 0) }} {{- if .Values.pulsar_metadata.configurationStore }} - - name: wait-cs-ready + - name: wait-zk-cs-ready image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}" imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}" resources: {{ toYaml .Values.initContainer.resources | nindent 10 }} @@ -54,34 +54,46 @@ spec: sleep 3; done; {{- end }} - - name: wait-zookeeper-ready + - name: wait-zk-metastore-ready image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}" imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}" resources: {{ toYaml .Values.initContainer.resources | nindent 10 }} command: ["timeout", "{{ .Values.pulsar_metadata.waitZookeeperTimeout }}", "sh", "-c"] args: - >- - {{- if $zk:=.Values.pulsar_metadata.userProvidedZookeepers }} + {{- if $zk := .Values.pulsar_metadata.userProvidedZookeepers }} export PULSAR_MEM="-Xmx128M"; until timeout 15 bin/pulsar zookeeper-shell -server {{ $zk }} ls {{ or .Values.metadataPrefix "/" }}; do echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3; done; - {{ else }} + {{ else if .Values.components.zookeeper }} until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}.{{ template "pulsar.namespace" . }}; do sleep 3; done; - {{- end}} + {{- end }} + {{- end }} + {{- if and .Values.components.oxia .Values.pulsar_metadata.waitOxiaTimeout (gt (.Values.pulsar_metadata.waitOxiaTimeout | int) 0) }} + - name: wait-oxia-ready + image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}" + imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}" + resources: {{ toYaml .Values.initContainer.resources | nindent 10 }} + command: ["timeout", "{{ .Values.pulsar_metadata.waitOxiaTimeout }}", "sh", "-c"] + args: + - >- + until nslookup {{ template "pulsar.oxia.server.service" . }}; do + sleep 3; + done; {{- end }} - {{- if and .Values.pulsar_metadata.waitBookkeeperTimeout (not (eq (.Values.pulsar_metadata.waitBookkeeperTimeout | toString) "0")) }} + {{- if and .Values.pulsar_metadata.waitBookkeeperTimeout (gt (.Values.pulsar_metadata.waitBookkeeperTimeout | int) 0) }} # This initContainer will wait for bookkeeper initnewcluster to complete # before initializing pulsar metadata - name: pulsar-bookkeeper-verify-clusterid - image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}" - imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}" + image: {{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }} + imagePullPolicy: {{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }} resources: {{ toYaml .Values.initContainer.resources | nindent 10 }} command: ["timeout", "{{ .Values.pulsar_metadata.waitBookkeeperTimeout }}", "sh", "-c"] args: - - > + - >- bin/apply-config-from-env.py conf/bookkeeper.conf; echo Default BOOKIE_MEM settings are set very high, which can cause the init container to fail.; echo Setting the memory to a lower value to avoid OOM as operations below are not memory intensive.; @@ -92,32 +104,47 @@ spec: done; envFrom: - configMapRef: - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}" + name: {{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }} volumeMounts: {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 8 }} {{- end }} containers: - - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }}" - image: "{{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }}" - imagePullPolicy: "{{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }}" + - name: {{ template "pulsar.fullname" . }}-{{ .Values.pulsar_metadata.component }} + image: {{ template "pulsar.imageFullName" (dict "image" .Values.pulsar_metadata.image "root" .) }} + imagePullPolicy: {{ template "pulsar.imagePullPolicy" (dict "image" .Values.pulsar_metadata.image "root" .) }} {{- if .Values.pulsar_metadata.resources }} resources: {{ toYaml .Values.pulsar_metadata.resources | indent 10 }} {{- end }} command: ["timeout", "{{ .Values.pulsar_metadata.initTimeout | default 60 }}", "sh", "-c"] + {{- if .Values.components.zookeeper }} args: - - | + - >- {{- include "pulsar.toolset.zookeeper.tls.settings" . | nindent 12 }} export PULSAR_MEM="-Xmx128M"; bin/pulsar initialize-cluster-metadata \ --cluster {{ template "pulsar.cluster.name" . }} \ --zookeeper {{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }} \ - {{- if .Values.pulsar_metadata.configurationStore }} + {{- if .Values.pulsar_metadata.configurationStore }} --configuration-store {{ template "pulsar.configurationStore.connect" . }}{{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }} \ - {{- end }} - {{- if not .Values.pulsar_metadata.configurationStore }} + {{- else }} --configuration-store {{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }} \ - {{- end }} + {{- end }} + --web-service-url http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.http }}/ \ + --web-service-url-tls https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.https }}/ \ + --broker-service-url pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.pulsar }}/ \ + --broker-service-url-tls pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.pulsarssl }}/ ; + {{- if .Values.extraInitCommand }} + {{ .Values.extraInitCommand }} + {{- end }} + {{- else if .Values.components.oxia }} + args: + - >- + export PULSAR_MEM="-Xmx128M"; + bin/pulsar initialize-cluster-metadata \ + --cluster {{ template "pulsar.cluster.name" . }} \ + --metadata-store "{{ template "pulsar.oxia.metadata.url.broker" . }}" \ + --configuration-store "{{ template "pulsar.oxia.metadata.url.broker" . }}" \ --web-service-url http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.http }}/ \ --web-service-url-tls https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.https }}/ \ --broker-service-url pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.ports.pulsar }}/ \ @@ -125,10 +152,11 @@ spec: {{- if .Values.extraInitCommand }} {{ .Values.extraInitCommand }} {{- end }} + {{- end }} volumeMounts: - {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 8 }} + {{- include "pulsar.toolset.certs.volumeMounts" . | nindent 10 }} volumes: - {{- include "pulsar.toolset.certs.volumes" . | nindent 6 }} + {{- include "pulsar.toolset.certs.volumes" . | nindent 8 }} restartPolicy: OnFailure {{- if .Values.pulsar_metadata.nodeSelector }} nodeSelector: diff --git a/charts/pulsar/templates/toolset-service-account.yaml b/charts/pulsar/templates/toolset-service-account.yaml index 32b0b044..290076f1 100644 --- a/charts/pulsar/templates/toolset-service-account.yaml +++ b/charts/pulsar/templates/toolset-service-account.yaml @@ -26,8 +26,8 @@ metadata: labels: {{- include "pulsar.standardLabels" . | nindent 4 }} component: {{ .Values.toolset.component }} - annotations: {{- with .Values.toolset.service_account.annotations }} + annotations: {{ toYaml . | indent 4 }} {{- end }} {{- end }} diff --git a/charts/pulsar/templates/zookeeper-podmonitor.yaml b/charts/pulsar/templates/zookeeper-podmonitor.yaml index f2de68b1..15929037 100644 --- a/charts/pulsar/templates/zookeeper-podmonitor.yaml +++ b/charts/pulsar/templates/zookeeper-podmonitor.yaml @@ -18,6 +18,7 @@ # # deploy zookeeper PodMonitor only when `$.Values.zookeeper.podMonitor.enabled` is true +{{- if .Values.components.zookeeper }} {{- if $.Values.zookeeper.podMonitor.enabled }} apiVersion: monitoring.coreos.com/v1 kind: PodMonitor @@ -56,3 +57,4 @@ spec: {{- include "pulsar.matchLabels" . | nindent 6 }} component: zookeeper {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/pulsar/values.yaml b/charts/pulsar/values.yaml index 6fe428c5..401d0894 100755 --- a/charts/pulsar/values.yaml +++ b/charts/pulsar/values.yaml @@ -113,6 +113,8 @@ affinity: components: # zookeeper zookeeper: true + # oxia + oxia: false # bookkeeper bookkeeper: true # bookkeeper - autorecovery @@ -202,6 +204,10 @@ images: # uses defaultPullPolicy when unspecified pullPolicy: hasCommand: false + oxia: + repository: streamnative/oxia + tag: 0.11.9 + pullPolicy: Always ## TLS ## templates/tls-certs.yaml @@ -249,6 +255,8 @@ tls: function_instance: # controls the use of TLS for function runtime connections towards brokers enabled: false + oxia: + enabled: false # Enable or disable broker authentication and authorization. auth: @@ -468,7 +476,81 @@ zookeeper: usePolicy: true maxUnavailable: 1 - +## Pulsar: Oxia cluster +oxia: + component: oxia + initialShardCount: 3 + replicationFactor: 3 +## templates/coordinator-deployment.yaml + coordinator: + # This is how prometheus discovers this component + podMonitor: + enabled: true + interval: 60s + scrapeTimeout: 60s + metricRelabelings: + # - action: labeldrop + # regex: cluster + cpuLimit: 100m + memoryLimit: 128Mi + ports: + internal: 6649 + metrics: 8080 + service: + annotations: {} + service_account: + annotations: {} + tolerations: [] + # nodeSelector: + # cloud.google.com/gke-nodepool: default-pool +## templates/server-statefulset.yaml + server: + # This is how prometheus discovers this component + podMonitor: + enabled: true + interval: 60s + scrapeTimeout: 60s + metricRelabelings: + # - action: labeldrop + # regex: cluster + replicas: 3 + # CPU limit for the server pod + cpuLimit: 1 + # Memory limit for the server pod + memoryLimit: 1Gi + # Oxia database cache size in MB + dbCacheSizeMb: 512 + # Storage size for the PVC of the server pod + storageSize: 8Gi + # Storage class name for the PVC of the server pod + # storageClassName: existent-storage-class + ports: + public: 6648 + internal: 6649 + metrics: 8080 + service: + public: + annotations: {} + internal: + annotations: {} + service_account: + annotations: {} + securityContext: + fsGroup: 0 + fsGroupChangePolicy: "OnRootMismatch" + affinity: + anti_affinity: true + anti_affinity_topology_key: kubernetes.io/hostname + # Set the anti affinity type. Valid values: + # requiredDuringSchedulingIgnoredDuringExecution - rules must be met for pod to be scheduled (hard) requires at least one node per replica + # preferredDuringSchedulingIgnoredDuringExecution - scheduler will try to enforce but not guranentee + type: requiredDuringSchedulingIgnoredDuringExecution + # set topologySpreadConstraint to deploy pods across different zones + topologySpreadConstraints: [] + tolerations: [] + # nodeSelector: + # cloud.google.com/gke-nodepool: default-pool + pprofEnabled: false ## Pulsar: Bookkeeper cluster ## templates/bookkeeper-statefulset.yaml ## @@ -483,6 +565,8 @@ bookkeeper: waitZookeeperTimeout: 600 ## Timeout for running metadata initialization initTimeout: 60 + ## Timeout for waiting for oxia to be available before running metadata initialization. This setting applies only when oxia is enabled. + waitOxiaTimeout: 600 ## Set the resources used for running `bin/bookkeeper shell initnewcluster` ## resources: @@ -765,12 +849,16 @@ pulsar_metadata: tag: # uses defaultPullPolicy when unspecified pullPolicy: - ## set an existing configuration store + ## set an existing configuration store. This setting applies only when zookeeper is enabled. # configurationStore: + # the prefix for the configuration store metadata. This setting applies only when zookeeper is enabled. configurationStoreMetadataPrefix: "" + # the configuration store port. This setting applies only when zookeeper is enabled. configurationStorePort: 2181 - ## Timeout for waiting for zookeeper to become available before running metadata initialization + # the zookeeper timeout. This setting applies only when zookeeper is enabled. waitZookeeperTimeout: 600 + ## Timeout for waiting for oxia to be available before running metadata initialization. This setting applies only when oxia is enabled. + waitOxiaTimeout: 600 ## Timeout for waiting for bookkeeper to be initialized before running metadata initialization waitBookkeeperTimeout: 120 ## Timeout for running metadata initialization @@ -866,6 +954,8 @@ broker: gracePeriod: 30 ## Timeout for waiting for zookeeper to become available before starting a broker waitZookeeperTimeout: 600 + ## Timeout for waiting for oxia to be available before starting a broker. This setting applies only when oxia is enabled. + waitOxiaTimeout: 600 ## Timeout for waiting for bookkeeper to become available before starting a broker waitBookkeeperTimeout: 120 resources: @@ -1106,6 +1196,8 @@ proxy: gracePeriod: 30 ## Timeout for waiting for zookeeper to become available before starting a proxy waitZookeeperTimeout: 600 + ## Timeout for waiting for oxia to be available before starting a proxy. This setting applies only when oxia is enabled. + waitOxiaTimeout: 600 ## Timeout for waiting for brokers to become available before starting a proxy waitBrokerTimeout: 120 resources: diff --git a/examples/values-oxia.yaml b/examples/values-oxia.yaml new file mode 100644 index 00000000..7c1ecb73 --- /dev/null +++ b/examples/values-oxia.yaml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +components: + zookeeper: false + oxia: true \ No newline at end of file