feat(helm): deprecated prerequisites chart and added init containers

README.md

@@ -76,7 +76,7 @@ You should get a result similar to below.
 ```
 NAME READY STATUS RESTARTS AGE
 elasticsearch-master-0 1/1 Running 0 62m
-prerequisites-cp-schema-registry-cf79bfccf-kvjtv 2/2 Running 1 63m
+prerequisites-schema-registry-cf79bfccf-kvjtv  2/2 Running 1 63m
 prerequisites-kafka-0 1/1 Running 2 62m
 prerequisites-mysql-0 1/1 Running 1 62m
 prerequisites-neo4j-community-0 1/1 Running 0 52m
@@ -106,7 +106,7 @@ datahub-elasticsearch-setup-job-8dz6b 0/1 Completed 0
 datahub-kafka-setup-job-6blcj 0/1 Completed 0 4m40s
 datahub-mysql-setup-job-b57kc 0/1 Completed 0 4m7s
 elasticsearch-master-0 1/1 Running 0 97m
-prerequisites-cp-schema-registry-cf79bfccf-kvjtv 2/2 Running 1 99m
+prerequisites-schema-registry-cf79bfccf-kvjtv  2/2 Running 1 99m
 prerequisites-kafka-0 1/1 Running 2 97m
 prerequisites-mysql-0 1/1 Running 1 97m
 prerequisites-neo4j-community-0 1/1 Running 0 88m

charts/datahub/Chart.yaml

@@ -4,13 +4,13 @@ description: A Helm chart for LinkedIn DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.163
+version: 0.2.164
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.10.2
 dependencies:
  - name: datahub-gms
- version: 0.2.146
+ version: 0.2.147
  repository: file://./subcharts/datahub-gms
  condition: datahub-gms.enabled
  - name: datahub-frontend

charts/datahub/README.md

@@ -73,6 +73,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | postgresqlSetupJob.podAnnotations | object | `{}` | Pod annotations for mysqlSetupJob |
 | global.strict_mode | boolean | true | Enables validations in helm charts to ensure features work as expected. Recommended NOT TO CHANGE. |
 | global.datahub_standalone_consumers_enabled | boolean | true | Enable standalone consumers for kafka |
+| global.helmHooks | boolean | true | Enable deployment with helm hooks |
 | global.datahub_analytics_enabled | boolean | true | Enable datahub usage analytics |
 | global.datahub.appVersion | string | `"1.0"` | App version for annotation |
 | global.datahub.gms.port | string | `"8080"` | Port of GMS service |
@@ -89,7 +90,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | global.kafka.topics.metadata_change_log_versioned_topic_name | string | `"MetadataChangeLog_Versioned_v1"` | Kafka topic name for Versioned Metadata Change Log events |
 | global.kafka.topics.metadata_change_log_timeseries_topic_name | string | `"MetadataChangeLog_Timeseries_v1"` | Kafka topic name for Timeseries Metadata Change Log events |
 | global.kafka.topics.platform_event_topic_name | string | `"PlatformEvent_v1"` | Kafka topic name for Platform events |
-| global.kafka.schemaregistry.url | string | `"http://prerequisites-cp-schema-registry:8081"` | URL to kafka schema registry |
+| global.kafka.schemaregistry.url | string | `"http://prerequisites-schema-registry:8081"` | URL to kafka schema registry |
 | global.neo4j.host | string | `"prerequisites-neo4j:7474"` | Neo4j host address (with port) |
 | global.neo4j.uri | string | `"bolt://prerequisites-neo4j"` | Neo4j URI |
 | global.neo4j.username | string | `"neo4j"` | Neo4j user name |

charts/datahub/quickstart-values-with-neo4j.yaml

@@ -78,7 +78,7 @@ global:
  zookeeper:
  server: "prerequisites-zookeeper:2181"
  schemaregistry:
- url: "http://prerequisites-cp-schema-registry:8081"
+ url: "http://prerequisites-schema-registry:8081"
 
  neo4j:
  host: "prerequisites-neo4j-community:7474"

charts/datahub/subcharts/acryl-datahub-actions/values.yaml

@@ -72,7 +72,7 @@ global:
  bootstrap:
  server: "broker:9092"
  schemaregistry:
- url: "http://prerequisites-cp-schema-registry:8081"
+ url: "http://prerequisites-schema-registry:8081"
 
  datahub:
  gms:

charts/datahub/subcharts/datahub-gms/Chart.yaml

@@ -12,7 +12,7 @@ description: A Helm chart for LinkedIn DataHub's datahub-gms component
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.146
+version: 0.2.147
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.10.0

charts/datahub/subcharts/datahub-gms/templates/deployment.yaml

@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
- name: {{ include "datahub-gms.fullname" . }}
+ name: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
  labels:
  {{- include "datahub-gms.labels" . | nindent 4 }}
  {{- range $key, $val := .Values.extraLabels }}
@@ -51,6 +51,9 @@ spec:
  {{- toYaml . | nindent 8 }}
  {{- end }}
  initContainers:
+ {{- if .Values.initContainers }}
+ {{- tpl (toYaml .Values.initContainers) . | nindent 6 }}
+ {{- end }}
  {{- with .Values.extraInitContainers }}
  {{- toYaml . | nindent 8 }}
  {{- end }}

charts/datahub/subcharts/datahub-gms/values.yaml

@@ -90,6 +90,8 @@ extraVolumeMounts: []
  # mountPath: /usr/share/extras
  # readOnly: true
 
+initContainers: []
+
 extraInitContainers: []
 
 resources: {}

charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml

@@ -3,7 +3,7 @@
 # Creates a suspended cronJob that you can use to create an adhoc job when ready to run clean up.
 # Run the following command to do so
 # kubectl create job --from=cronjob/<<release-name>>-datahub-cleanup-job-template datahub-cleanup-job
-apiVersion: {{ include "datahub.cronjob.apiVersion" $}}
+apiVersion: {{ include "datahub.cronjob.apiVersion" $ }}
 kind: CronJob
 metadata:
  name: {{ .Release.Name }}-datahub-cleanup-job-template
@@ -60,7 +60,7 @@ spec:
  imagePullPolicy: {{ .Values.datahubUpgrade.image.pullPolicy | default "IfNotPresent" }}
  args: [ "-u", "NoCodeDataMigrationCleanup" ]
  env:
- {{- include "datahub.upgrade.env" . | nindent 16}}
+ {{- include "datahub.upgrade.env" . | nindent 16 }}
  {{- with .Values.datahubUpgrade.extraEnvs }}
  {{- toYaml . | nindent 16 }}
  {{- end }}

charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml

@@ -8,15 +8,17 @@ metadata:
  app.kubernetes.io/instance: {{ .Release.Name | quote }}
  app.kubernetes.io/version: {{ .Chart.AppVersion }}
  helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ {{- if .Values.global.helmHooks }}
  annotations:
  # This is what defines this resource as a hook. Without this line, the
  # job is considered part of the release.
  "helm.sh/hook": post-install,post-upgrade
  "helm.sh/hook-weight": "-2"
  "helm.sh/hook-delete-policy": before-hook-creation
+ {{- end }}
 spec:
  template:
- {{- if or .Values.global.podLabels .Values.datahubUpgrade.podAnnotations}}
+ {{- if or .Values.global.podLabels .Values.datahubUpgrade.podAnnotations }}
  metadata:
  {{- with .Values.datahubUpgrade.podAnnotations }}
  annotations:
@@ -55,8 +57,8 @@ spec:
  securityContext:
  {{- toYaml .Values.datahubUpgrade.podSecurityContext | nindent 8 }}
  initContainers:
- {{- with .Values.datahubUpgrade.extraInitContainers }}
-  {{- toYaml . | nindent 12 }}
+ {{- if .Values.datahubUpgrade.initContainers }}
+ {{-  tpl (toYaml .Values.datahubUpgrade.initContainers) . | nindent 6 }}
  {{- end }}
  containers:
  - name: datahub-upgrade-job
@@ -72,7 +74,7 @@ spec:
  - "-a"
  - "dbType={{ .Values.datahubUpgrade.noCodeDataMigration.sqlDbType }}"
  env:
- {{- include "datahub.upgrade.env" . | nindent 12}}
+ {{- include "datahub.upgrade.env" . | nindent 12 }}
  {{- with .Values.datahubUpgrade.extraEnvs }}
  {{- toYaml . | nindent 12 }}
  {{- end }}

charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml

@@ -72,7 +72,7 @@ spec:
  - "-a"
  - "batchDelayMs={{ .Values.datahubUpgrade.batchDelayMs }}"
  env:
- {{- include "datahub.upgrade.env" . | nindent 16}}
+ {{- include "datahub.upgrade.env" . | nindent 16 }}
  {{- with .Values.datahubUpgrade.extraEnvs }}
  {{- toYaml . | nindent 16 }}
  {{- end }}
@@ -87,7 +87,7 @@ spec:
  {{- toYaml . | nindent 16 }}
  {{- end }}
  resources:
- {{- toYaml .Values.datahubUpgrade.restoreIndices.resources | nindent 16}}
+ {{- toYaml .Values.datahubUpgrade.restoreIndices.resources | nindent 16 }}
  {{- with .Values.datahubUpgrade.nodeSelector }}
  nodeSelector:
  {{- toYaml . | nindent 12 }}

charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml

@@ -8,15 +8,17 @@ metadata:
  app.kubernetes.io/instance: {{ .Release.Name | quote }}
  app.kubernetes.io/version: {{ .Chart.AppVersion }}
  helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ {{- if .Values.global.helmHooks }}
  annotations:
  # This is what defines this resource as a hook. Without this line, the
  # job is considered part of the release.
  "helm.sh/hook": pre-install,pre-upgrade
  "helm.sh/hook-weight": "-4"
  "helm.sh/hook-delete-policy": before-hook-creation
+ {{- end }}
 spec:
  template:
- {{- if or .Values.global.podLabels .Values.datahubSystemUpdate.podAnnotations}}
+ {{- if or .Values.global.podLabels .Values.datahubSystemUpdate.podAnnotations }}
  metadata:
  {{- with .Values.datahubSystemUpdate.podAnnotations }}
  annotations:
@@ -55,8 +57,8 @@ spec:
  securityContext:
  {{- toYaml .Values.datahubSystemUpdate.podSecurityContext | nindent 8 }}
  initContainers:
- {{- with .Values.datahubSystemUpdate.extraInitContainers }}
-  {{- toYaml . | nindent 12 }}
+ {{- if .Values.datahubSystemUpdate.initContainers }}
+ {{-  tpl (toYaml .Values.datahubSystemUpdate.initContainers) . | nindent 6 }}
  {{- end }}
  containers:
  - name: datahub-system-update-job
@@ -68,7 +70,7 @@ spec:
  env:
  - name: DATAHUB_REVISION
  value: {{ .Release.Revision | quote }}
- {{- include "datahub.upgrade.env" . | nindent 12}}
+ {{- include "datahub.upgrade.env" . | nindent 12 }}
  - name: DATAHUB_ANALYTICS_ENABLED
  value: {{ .Values.global.datahub_analytics_enabled | quote }}
  {{- with .Values.global.kafka.schemaregistry.type }}

charts/datahub/templates/elasticsearch-setup-job.yml

@@ -8,12 +8,14 @@ metadata:
  app.kubernetes.io/instance: {{ .Release.Name | quote }}
  app.kubernetes.io/version: {{ .Chart.AppVersion }}
  helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ {{- if .Values.global.helmHooks }}
  annotations:
  # This is what defines this resource as a hook. Without this line, the
  # job is considered part of the release.
  "helm.sh/hook": pre-install,pre-upgrade
  "helm.sh/hook-weight": "-5"
  "helm.sh/hook-delete-policy": before-hook-creation
+ {{- end }}
 spec:
  template:
  {{- if or .Values.global.podLabels .Values.elasticsearchSetupJob.podAnnotations }}
@@ -48,6 +50,10 @@ spec:
  restartPolicy: Never
  securityContext:
  {{- toYaml .Values.elasticsearchSetupJob.podSecurityContext | nindent 8 }}
+ initContainers:
+ {{- if .Values.elasticsearchSetupJob.initContainers }}
+ {{- tpl (toYaml .Values.elasticsearchSetupJob.initContainers) . | nindent 6 }}
+ {{- end }}
  containers:
  - name: elasticsearch-setup-job
  image: "{{ .Values.elasticsearchSetupJob.image.repository }}:{{ required "Global or specific tag is required" ( .Values.elasticsearchSetupJob.image.tag | default .Values.global.datahub.version) }}"

charts/datahub/templates/kafka-setup-job.yml

@@ -8,12 +8,14 @@ metadata:
  app.kubernetes.io/instance: {{ .Release.Name | quote }}
  app.kubernetes.io/version: {{ .Chart.AppVersion }}
  helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ {{- if .Values.global.helmHooks }}
  annotations:
  # This is what defines this resource as a hook. Without this line, the
  # job is considered part of the release.
  "helm.sh/hook": pre-install,pre-upgrade
  "helm.sh/hook-weight": "-5"
  "helm.sh/hook-delete-policy": before-hook-creation
+ {{- end }}
 spec:
  template:
  {{- if or .Values.global.podLabels .Values.kafkaSetupJob.podAnnotations }}
@@ -54,6 +56,10 @@ spec:
  {{- with .Values.kafkaSetupJob.extraVolumes }}
  {{- toYaml . | nindent 8 }}
  {{- end }}
+ initContainers:
+ {{- if .Values.kafkaSetupJob.initContainers }}
+ {{- tpl (toYaml .Values.kafkaSetupJob.initContainers) . | nindent 6 }}
+ {{- end }}
  containers:
  - name: kafka-setup-job
  image: "{{ .Values.kafkaSetupJob.image.repository }}:{{ required "Global or specific tag is required" ( .Values.kafkaSetupJob.image.tag | default .Values.global.datahub.version) }}"

charts/datahub/templates/mysql-setup-job.yml

@@ -8,12 +8,14 @@ metadata:
  app.kubernetes.io/instance: {{ .Release.Name | quote }}
  app.kubernetes.io/version: {{ .Chart.AppVersion }}
  helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ {{- if .Values.global.helmHooks }}
  annotations:
  # This is what defines this resource as a hook. Without this line, the
  # job is considered part of the release.
  "helm.sh/hook": pre-install,pre-upgrade
  "helm.sh/hook-weight": "-5"
  "helm.sh/hook-delete-policy": before-hook-creation
+ {{- end }}
 spec:
  template:
  {{- if or .Values.global.podLabels .Values.mysqlSetupJob.podAnnotations }}
@@ -48,6 +50,10 @@ spec:
  restartPolicy: Never
  securityContext:
  {{- toYaml .Values.mysqlSetupJob.podSecurityContext | nindent 8 }}
+ initContainers:
+ {{- if .Values.mysqlSetupJob.initContainers }}
+ {{- tpl (toYaml .Values.mysqlSetupJob.initContainers) . | nindent 6 }}
+ {{- end }}
  containers:
  - name: mysql-setup-job
  image: "{{ .Values.mysqlSetupJob.image.repository }}:{{ required "Global or specific tag is required" ( .Values.mysqlSetupJob.image.tag | default .Values.global.datahub.version) }}"

charts/datahub/templates/postgresql-setup-job.yml

@@ -8,12 +8,14 @@ metadata:
  app.kubernetes.io/instance: {{ .Release.Name | quote }}
  app.kubernetes.io/version: {{ .Chart.AppVersion }}
  helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ {{- if .Values.global.helmHooks }}
  annotations:
  # This is what defines this resource as a hook. Without this line, the
  # job is considered part of the release.
  "helm.sh/hook": pre-install,pre-upgrade
  "helm.sh/hook-weight": "-5"
  "helm.sh/hook-delete-policy": before-hook-creation
+ {{- end }}
 spec:
  template:
  {{- if or .Values.global.podLabels .Values.postgresqlSetupJob.podAnnotations }}
@@ -48,6 +50,10 @@ spec:
  restartPolicy: Never
  securityContext:
  {{- toYaml .Values.postgresqlSetupJob.podSecurityContext | nindent 8 }}
+ initContainers:
+ {{- if .Values.postgresqlSetupJob.initContainers }}
+ {{- tpl (toYaml .Values.postgresqlSetupJob.initContainers) . | nindent 6 }}
+ {{- end }}
  containers:
  - name: postgresql-setup-job
  image: "{{ .Values.postgresqlSetupJob.image.repository }}:{{ required "Global or specific tag is required" (.Values.postgresqlSetupJob.image.tag | default .Values.global.datahub.version) }}"

charts/datahub/templates/rolebinding.yml

@@ -0,0 +1,25 @@
+{{- if .Values.global.helmHooks }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ .Release.Name }}-reader
+rules:
+- apiGroups: ["batch"]
+ resources: ["jobs"]
+ verbs: ["get", "list", "watch"]
+{{- end }}
+---
+{{- if .Values.global.helmHooks }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ .Release.Name }}-reader-binding
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ .Release.Name }}-reader
+subjects:
+- kind: ServiceAccount
+ name: datahub-gms
+ namespace: {{ .Release.Namespace }}
+{{- end }}