feat(helm): deprecated prerequisites chart and added init containers

README.md

@@ -76,7 +76,7 @@ You should get a result similar to below.
 ```
 NAME                                               READY   STATUS      RESTARTS   AGE
 elasticsearch-master-0                             1/1     Running     0          62m
-prerequisites-cp-schema-registry-cf79bfccf-kvjtv   2/2     Running     1          63m
+prerequisites-schema-registry-cf79bfccf-kvjtv      2/2     Running     1          63m
 prerequisites-kafka-0                              1/1     Running     2          62m
 prerequisites-mysql-0                              1/1     Running     1          62m
 prerequisites-neo4j-community-0                    1/1     Running     0          52m
@@ -106,7 +106,7 @@ datahub-elasticsearch-setup-job-8dz6b              0/1     Completed   0
 datahub-kafka-setup-job-6blcj                      0/1     Completed   0          4m40s
 datahub-mysql-setup-job-b57kc                      0/1     Completed   0          4m7s
 elasticsearch-master-0                             1/1     Running     0          97m
-prerequisites-cp-schema-registry-cf79bfccf-kvjtv   2/2     Running     1          99m
+prerequisites-schema-registry-cf79bfccf-kvjtv      2/2     Running     1          99m
 prerequisites-kafka-0                              1/1     Running     2          97m
 prerequisites-mysql-0                              1/1     Running     1          97m
 prerequisites-neo4j-community-0                    1/1     Running     0          88m

charts/datahub/Chart.yaml

@@ -4,13 +4,13 @@ description: A Helm chart for LinkedIn DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.163
+version: 0.2.164
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.10.2
 dependencies:
   - name: datahub-gms
-    version: 0.2.146
+    version: 0.2.147
     repository: file://./subcharts/datahub-gms
     condition: datahub-gms.enabled
   - name: datahub-frontend

charts/datahub/README.md

@@ -73,6 +73,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | postgresqlSetupJob.podAnnotations | object | `{}` | Pod annotations for mysqlSetupJob |
 | global.strict_mode | boolean | true | Enables validations in helm charts to ensure features work as expected. Recommended NOT TO CHANGE. |
 | global.datahub_standalone_consumers_enabled | boolean | true | Enable standalone consumers for kafka |
+| global.helmHooks | boolean | true | Enable deployment with helm hooks |
 | global.datahub_analytics_enabled | boolean | true | Enable datahub usage analytics |
 | global.datahub.appVersion | string | `"1.0"` | App version for annotation |
 | global.datahub.gms.port | string | `"8080"` | Port of GMS service |
@@ -89,7 +90,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | global.kafka.topics.metadata_change_log_versioned_topic_name | string | `"MetadataChangeLog_Versioned_v1"` | Kafka topic name for Versioned Metadata Change Log events |
 | global.kafka.topics.metadata_change_log_timeseries_topic_name | string | `"MetadataChangeLog_Timeseries_v1"` | Kafka topic name for Timeseries Metadata Change Log events |
 | global.kafka.topics.platform_event_topic_name | string | `"PlatformEvent_v1"` | Kafka topic name for Platform events |
-| global.kafka.schemaregistry.url | string | `"http://prerequisites-cp-schema-registry:8081"` | URL to kafka schema registry |
+| global.kafka.schemaregistry.url | string | `"http://prerequisites-schema-registry:8081"` | URL to kafka schema registry |
 | global.neo4j.host | string | `"prerequisites-neo4j:7474"` | Neo4j host address (with port) |
 | global.neo4j.uri | string | `"bolt://prerequisites-neo4j"` | Neo4j URI |
 | global.neo4j.username | string | `"neo4j"` | Neo4j user name |

charts/datahub/quickstart-values-with-neo4j.yaml

@@ -78,7 +78,7 @@ global:
     zookeeper:
       server: "prerequisites-zookeeper:2181"
     schemaregistry:
-      url: "http://prerequisites-cp-schema-registry:8081"
+      url: "http://prerequisites-schema-registry:8081"
 
   neo4j:
     host: "prerequisites-neo4j-community:7474"

charts/datahub/subcharts/acryl-datahub-actions/values.yaml

@@ -72,7 +72,7 @@ global:
     bootstrap:
       server: "broker:9092"
     schemaregistry:
-      url: "http://prerequisites-cp-schema-registry:8081"
+      url: "http://prerequisites-schema-registry:8081"
 
   datahub:
     gms:

charts/datahub/subcharts/datahub-gms/Chart.yaml

@@ -12,7 +12,7 @@ description: A Helm chart for LinkedIn DataHub's datahub-gms component
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.146
+version: 0.2.147
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.10.0

charts/datahub/subcharts/datahub-gms/templates/deployment.yaml

@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ include "datahub-gms.fullname" . }}
+  name: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
   labels:
     {{- include "datahub-gms.labels" . | nindent 4 }}
     {{- range $key, $val := .Values.extraLabels }}
@@ -51,6 +51,9 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       initContainers:
+      {{- if .Values.initContainers }}
+      {{-  tpl (toYaml .Values.initContainers) . | nindent 6 }}
+      {{- end }}
       {{- with .Values.extraInitContainers }}
         {{- toYaml . | nindent 8 }}
       {{- end }}

charts/datahub/subcharts/datahub-gms/values.yaml

@@ -90,6 +90,8 @@ extraVolumeMounts: []
   #   mountPath: /usr/share/extras
   #   readOnly: true
 
+initContainers: []
+
 extraInitContainers: []
 
 resources: {}

charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml

@@ -3,7 +3,7 @@
 # Creates a suspended cronJob that you can use to create an adhoc job when ready to run clean up.
 # Run the following command to do so
 # kubectl create job --from=cronjob/<<release-name>>-datahub-cleanup-job-template datahub-cleanup-job
-apiVersion: {{ include "datahub.cronjob.apiVersion" $}}
+apiVersion: {{ include "datahub.cronjob.apiVersion" $ }}
 kind: CronJob
 metadata:
   name: {{ .Release.Name }}-datahub-cleanup-job-template
@@ -60,7 +60,7 @@ spec:
               imagePullPolicy: {{ .Values.datahubUpgrade.image.pullPolicy | default "IfNotPresent" }}
               args: [ "-u", "NoCodeDataMigrationCleanup" ]
               env:
-                {{- include "datahub.upgrade.env" . | nindent 16}}
+                {{- include "datahub.upgrade.env" . | nindent 16 }}
               {{- with .Values.datahubUpgrade.extraEnvs }}
                 {{- toYaml . | nindent 16 }}
               {{- end }}

charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml

@@ -8,15 +8,17 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name | quote }}
     app.kubernetes.io/version: {{ .Chart.AppVersion }}
     helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  {{- if .Values.global.helmHooks }}
   annotations:
     # This is what defines this resource as a hook. Without this line, the
     # job is considered part of the release.
     "helm.sh/hook": post-install,post-upgrade
     "helm.sh/hook-weight": "-2"
     "helm.sh/hook-delete-policy": before-hook-creation
+  {{- end }}
 spec:
   template:
-    {{- if or .Values.global.podLabels .Values.datahubUpgrade.podAnnotations}}
+    {{- if or .Values.global.podLabels .Values.datahubUpgrade.podAnnotations }}
     metadata:
     {{- with .Values.datahubUpgrade.podAnnotations }}
       annotations:
@@ -55,8 +57,8 @@ spec:
       securityContext:
         {{- toYaml .Values.datahubUpgrade.podSecurityContext | nindent 8 }}
       initContainers:
-      {{- with .Values.datahubUpgrade.extraInitContainers }}
-        {{- toYaml . | nindent 12 }}
+      {{- if .Values.datahubUpgrade.initContainers }}
+      {{-  tpl (toYaml .Values.datahubUpgrade.initContainers) . | nindent 6 }}
       {{- end }}
       containers:
         - name: datahub-upgrade-job
@@ -72,7 +74,7 @@ spec:
             - "-a"
             - "dbType={{ .Values.datahubUpgrade.noCodeDataMigration.sqlDbType }}"
           env:
-            {{- include "datahub.upgrade.env" . | nindent 12}}
+            {{- include "datahub.upgrade.env" . | nindent 12 }}
           {{- with .Values.datahubUpgrade.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml

@@ -72,7 +72,7 @@ spec:
                 - "-a"
                 - "batchDelayMs={{ .Values.datahubUpgrade.batchDelayMs }}"
               env:
-                {{- include "datahub.upgrade.env" . | nindent 16}}
+                {{- include "datahub.upgrade.env" . | nindent 16 }}
               {{- with .Values.datahubUpgrade.extraEnvs }}
                 {{- toYaml . | nindent 16 }}
               {{- end }}
@@ -87,7 +87,7 @@ spec:
                 {{- toYaml . | nindent 16 }}
               {{- end }}
               resources:
-                {{- toYaml .Values.datahubUpgrade.restoreIndices.resources | nindent 16}}
+                {{- toYaml .Values.datahubUpgrade.restoreIndices.resources | nindent 16 }}
           {{- with .Values.datahubUpgrade.nodeSelector }}
           nodeSelector:
             {{- toYaml . | nindent 12 }}

charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml

@@ -8,15 +8,17 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name | quote }}
     app.kubernetes.io/version: {{ .Chart.AppVersion }}
     helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  {{- if .Values.global.helmHooks }}
   annotations:
     # This is what defines this resource as a hook. Without this line, the
     # job is considered part of the release.
     "helm.sh/hook": pre-install,pre-upgrade
     "helm.sh/hook-weight": "-4"
     "helm.sh/hook-delete-policy": before-hook-creation
+  {{- end }}
 spec:
   template:
-    {{- if or .Values.global.podLabels .Values.datahubSystemUpdate.podAnnotations}}
+    {{- if or .Values.global.podLabels .Values.datahubSystemUpdate.podAnnotations }}
     metadata:
     {{- with .Values.datahubSystemUpdate.podAnnotations }}
       annotations:
@@ -55,8 +57,8 @@ spec:
       securityContext:
         {{- toYaml .Values.datahubSystemUpdate.podSecurityContext | nindent 8 }}
       initContainers:
-      {{- with .Values.datahubSystemUpdate.extraInitContainers }}
-        {{- toYaml . | nindent 12 }}
+      {{- if .Values.datahubSystemUpdate.initContainers }}
+      {{-  tpl (toYaml .Values.datahubSystemUpdate.initContainers) . | nindent 6 }}
       {{- end }}
       containers:
         - name: datahub-system-update-job
@@ -68,7 +70,7 @@ spec:
           env:
             - name: DATAHUB_REVISION
               value: {{ .Release.Revision | quote }}
-            {{- include "datahub.upgrade.env" . | nindent 12}}
+            {{- include "datahub.upgrade.env" . | nindent 12 }}
             - name: DATAHUB_ANALYTICS_ENABLED
               value: {{ .Values.global.datahub_analytics_enabled | quote }}
             {{- with .Values.global.kafka.schemaregistry.type }}

charts/datahub/templates/elasticsearch-setup-job.yml

@@ -8,12 +8,14 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name | quote }}
     app.kubernetes.io/version: {{ .Chart.AppVersion }}
     helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  {{- if .Values.global.helmHooks }}
   annotations:
     # This is what defines this resource as a hook. Without this line, the
     # job is considered part of the release.
     "helm.sh/hook": pre-install,pre-upgrade
     "helm.sh/hook-weight": "-5"
     "helm.sh/hook-delete-policy": before-hook-creation
+  {{- end }}
 spec:
   template:
     {{- if or .Values.global.podLabels .Values.elasticsearchSetupJob.podAnnotations }}
@@ -48,6 +50,10 @@ spec:
       restartPolicy: Never
       securityContext:
         {{- toYaml .Values.elasticsearchSetupJob.podSecurityContext | nindent 8 }}
+      initContainers:
+      {{- if .Values.elasticsearchSetupJob.initContainers }}
+      {{-  tpl (toYaml .Values.elasticsearchSetupJob.initContainers) . | nindent 6 }}
+      {{- end }}
       containers:
         - name: elasticsearch-setup-job
           image: "{{ .Values.elasticsearchSetupJob.image.repository }}:{{ required "Global or specific tag is required" ( .Values.elasticsearchSetupJob.image.tag | default .Values.global.datahub.version) }}"

charts/datahub/templates/kafka-setup-job.yml

@@ -8,12 +8,14 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name | quote }}
     app.kubernetes.io/version: {{ .Chart.AppVersion }}
     helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  {{- if .Values.global.helmHooks }}
   annotations:
     # This is what defines this resource as a hook. Without this line, the
     # job is considered part of the release.
     "helm.sh/hook": pre-install,pre-upgrade
     "helm.sh/hook-weight": "-5"
     "helm.sh/hook-delete-policy": before-hook-creation
+  {{- end }}
 spec:
   template:
     {{- if or .Values.global.podLabels .Values.kafkaSetupJob.podAnnotations }}
@@ -54,6 +56,10 @@ spec:
       {{- with .Values.kafkaSetupJob.extraVolumes }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      initContainers:
+      {{- if .Values.kafkaSetupJob.initContainers }}
+      {{-  tpl (toYaml .Values.kafkaSetupJob.initContainers) . | nindent 6 }}
+      {{- end }}
       containers:
         - name: kafka-setup-job
           image: "{{ .Values.kafkaSetupJob.image.repository }}:{{ required "Global or specific tag is required" ( .Values.kafkaSetupJob.image.tag | default .Values.global.datahub.version) }}"

charts/datahub/templates/mysql-setup-job.yml

@@ -8,12 +8,14 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name | quote }}
     app.kubernetes.io/version: {{ .Chart.AppVersion }}
     helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  {{- if .Values.global.helmHooks }}
   annotations:
     # This is what defines this resource as a hook. Without this line, the
     # job is considered part of the release.
     "helm.sh/hook": pre-install,pre-upgrade
     "helm.sh/hook-weight": "-5"
     "helm.sh/hook-delete-policy": before-hook-creation
+  {{- end }}
 spec:
   template:
     {{- if or .Values.global.podLabels .Values.mysqlSetupJob.podAnnotations }}
@@ -48,6 +50,10 @@ spec:
       restartPolicy: Never
       securityContext:
         {{- toYaml .Values.mysqlSetupJob.podSecurityContext | nindent 8 }}
+      initContainers:
+      {{- if .Values.mysqlSetupJob.initContainers }}
+      {{-  tpl (toYaml .Values.mysqlSetupJob.initContainers) . | nindent 6 }}
+      {{- end }}
       containers:
         - name: mysql-setup-job
           image: "{{ .Values.mysqlSetupJob.image.repository }}:{{ required "Global or specific tag is required" ( .Values.mysqlSetupJob.image.tag | default .Values.global.datahub.version) }}"

charts/datahub/templates/postgresql-setup-job.yml

@@ -8,12 +8,14 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name | quote }}
     app.kubernetes.io/version: {{ .Chart.AppVersion }}
     helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  {{- if .Values.global.helmHooks }}
   annotations:
     # This is what defines this resource as a hook. Without this line, the
     # job is considered part of the release.
     "helm.sh/hook": pre-install,pre-upgrade
     "helm.sh/hook-weight": "-5"
     "helm.sh/hook-delete-policy": before-hook-creation
+  {{- end }}
 spec:
   template:
     {{- if or .Values.global.podLabels .Values.postgresqlSetupJob.podAnnotations }}
@@ -48,6 +50,10 @@ spec:
       restartPolicy: Never
       securityContext:
         {{- toYaml .Values.postgresqlSetupJob.podSecurityContext | nindent 8 }}
+      initContainers:
+      {{- if .Values.postgresqlSetupJob.initContainers }}
+      {{-  tpl (toYaml .Values.postgresqlSetupJob.initContainers) . | nindent 6 }}
+      {{- end }}
       containers:
         - name: postgresql-setup-job
           image: "{{ .Values.postgresqlSetupJob.image.repository }}:{{ required "Global or specific tag is required" (.Values.postgresqlSetupJob.image.tag | default .Values.global.datahub.version) }}"

charts/datahub/templates/rolebinding.yml

@@ -0,0 +1,25 @@
+{{- if .Values.global.helmHooks }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}-reader
+rules:
+- apiGroups: ["batch"]
+  resources: ["jobs"]
+  verbs: ["get", "list", "watch"]
+{{- end }}
+---
+{{- if .Values.global.helmHooks }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Release.Name }}-reader-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}-reader
+subjects:
+- kind: ServiceAccount
+  name: datahub-gms
+  namespace: {{ .Release.Namespace }}
+{{- end }}