feat: created cronjob for reindex (#290)

Co-authored-by: Vatsal <safayavatsal@Vatsals-MacBook-Pro.local>
open-metadata · Dec 9, 2024 · d2318bb · d2318bb
1 parent ada02f3
commit d2318bb
Show file tree

Hide file tree

Showing 5 changed files with 160 additions and 0 deletions.
diff --git a/charts/openmetadata/README.md b/charts/openmetadata/README.md
@@ -224,6 +224,8 @@ helm install openmetadata open-metadata/openmetadata --values <<path-to-values-f
 | openmetadata.config.smtpConfig.username | string | `Empty String` | SMTP_SERVER_USERNAME |
 | openmetadata.config.upgradeMigrationConfigs.debug | bool | `false` |  |
 | openmetadata.config.upgradeMigrationConfigs.additionalArgs | string | `Empty String` |  |
+| openmetadata.config.reindexConfig.debug | bool | `false` |  |
+| openmetadata.config.reindexConfig.additionalArgs | string | `Empty String` |  |
 | openmetadata.config.web.enabled | bool | `true` | |
 | openmetadata.config.web.contentTypeOptions.enabled | bool | `false` | WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED |
 | openmetadata.config.web.csp.enabled | bool | `false` | WEB_CONF_XSS_CSP_ENABLED |

diff --git a/charts/openmetadata/templates/_helpers.tpl b/charts/openmetadata/templates/_helpers.tpl
@@ -351,3 +351,15 @@ OpenMetadata Configurations Environment Variables*/}}
 {{- end }}
 {{- end }}
 {{- end }}
+
+{{/*
+Build the OpenMetadata Deploy Pipelines Command using reindexConfig */}}
+{{- define "OpenMetadata.buildReindexCommand" }}
+  - "/bin/bash"
+  - "-c"
+  {{- if .Values.openmetadata.config.reindexConfig.debug }}
+  - "/opt/openmetadata/bootstrap/openmetadata-ops.sh -d deploy-pipelines {{ default "" .Values.openmetadata.config.reindexConfig.additionalArgs }}"
+  {{- else }}
+  - "/opt/openmetadata/bootstrap/openmetadata-ops.sh deploy-pipelines {{ default "" .Values.openmetadata.config.reindexConfig.additionalArgs }}"
+  {{- end }}
+{{- end }}
diff --git a/charts/openmetadata/templates/cron-reindex.yaml b/charts/openmetadata/templates/cron-reindex.yaml
@@ -0,0 +1,130 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cron-reindex
+  labels: 
+    {{- include "OpenMetadata.labels" . | indent 4 }}
+    {{- with .Values.deploymentAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 4 }}  
+    {{- end }}
+spec:
+  suspend: true
+  failedJobsHistoryLimit: 1  
+  successfulJobsHistoryLimit: 1 
+  jobTemplate:
+    metadata:                    
+      name: cron-reindex
+    spec:
+      selector:
+      matchLabels: 
+      {{- include "OpenMetadata.selectorLabels" . | nindent 6 }}
+      template:
+        metadata:
+          {{- with .Values.podAnnotations }}
+          annotations:
+            {{- toYaml . | nindent 8 }}
+          {{- end }}
+          labels:
+            {{- include "OpenMetadata.labels" . | indent 8 }}
+        spec:
+          {{- with .Values.imagePullSecrets  }}
+          imagePullSecrets:
+            {{- toYaml . | nindent 8 }}
+          {{- end }}
+          serviceAccountName: {{ include "OpenMetadata.serviceAccountName" . }}
+          {{- if not (.Values.automountServiceAccountToken) }}
+          automountServiceAccountToken: {{ .Values.automountServiceAccountToken}}
+          {{- end }}
+          securityContext:  
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          containers:
+            - name: cron-reindex
+              securityContext:
+                {{- toYaml .Values.podSecurityContext | nindent 16 }}
+              image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              volumeMounts:
+              {{- with .Values.extraVolumeMounts }}
+                {{- toYaml . | nindent 10 }}
+              {{- end }}
+              command: 
+                {{ include "OpenMetadata.buildReindexCommand" . | nindent 16 }}
+              env:
+              {{- include "OpenMetadata.configs" . | nindent 16 }}
+              {{- with .Values.extraEnvs }}
+                {{- toYaml . | nindent 10 }}
+              {{- end }}
+              envFrom:
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-config-secret
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-omd-secret
+                      {{- if .Values.openmetadata.config.database.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-db-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.elasticsearch.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-search-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.authorizer.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-authorizer-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.secretsManager.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-secretsmanager-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.web.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-web-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.authentication.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-authentication-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.eventMonitor.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-eventmonitor-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.pipelineServiceClientConfig.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-pipeline-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.smtpConfig.enableSmtpServer }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-smtp-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.jwtTokenConfiguration.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-jwt-secret
+                      {{- end }}
+                      {{- with .Values.openmetadata.config.fernetkey }}
+                      {{- if not .secretRef }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" $ }}-fernetkey-secret
+                      {{- end }}
+                      {{- end }}
+                      {{- with .Values.envFrom }}
+                        {{- toYaml . | nindent 10 }}
+                      {{- end }}
+              resources:
+                {{- toYaml .Values.resources | nindent 16 }}
+            {{- if .Values.sidecars }}
+              {{- include "tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 8 }}
+            {{- end }}
+          {{- with .Values.nodeSelector }}
+          nodeSelector:
+            {{- toYaml . | nindent 8 }}
+          {{- end }}  
+          {{- with .Values.affinity }}
+          affinity:
+            {{- toYaml . | nindent 8 }}
+          {{- end }}
+          {{- with .Values.tolerations }}
+          tolerations:
+            {{- toYaml . | nindent 8 }}
+          {{- end }}                     
+          restartPolicy: OnFailure
+  schedule: "0/5 * * * *"
diff --git a/charts/openmetadata/values.schema.json b/charts/openmetadata/values.schema.json
@@ -167,6 +167,18 @@
                                 }
                             }
                         },
+                        "reindexConfig": {
+                            "type": "object",
+                            "additionalProperties": false,
+                            "properties": {
+                                "additionalArgs": {
+                                    "type": "string"
+                                },
+                                "debug": {
+                                    "type": "boolean"
+                                }
+                            }
+                        },
                         "pipelineServiceClientConfig": {
                             "type": "object",
                             "additionalProperties": false,

diff --git a/charts/openmetadata/values.yaml b/charts/openmetadata/values.yaml
@@ -12,6 +12,10 @@ openmetadata:
       # You can pass the additional argument flags to the openmetadata-ops.sh migrate command
       # Example if you want to force migration runs, use additionalArgs: "--force"
       additionalArgs: ""
+    reindexConfig:
+      debug: false
+      # You can pass the additional argument flags to the openmetadata-ops.sh reindex command
+      additionalArgs: ""
     # Values can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL
     logLevel: INFO
     clusterName: openmetadata