Merge branch 'master' into job-annotations

README.md

@@ -76,8 +76,6 @@ You should get a result similar to below.
 ```
 NAME                                               READY   STATUS      RESTARTS   AGE
 elasticsearch-master-0                             1/1     Running     0          62m
-elasticsearch-master-1                             1/1     Running     0          62m
-elasticsearch-master-2                             1/1     Running     0          62m
 prerequisites-cp-schema-registry-cf79bfccf-kvjtv   2/2     Running     1          63m
 prerequisites-kafka-0                              1/1     Running     2          62m
 prerequisites-mysql-0                              1/1     Running     1          62m
@@ -108,8 +106,6 @@ datahub-elasticsearch-setup-job-8dz6b              0/1     Completed   0
 datahub-kafka-setup-job-6blcj                      0/1     Completed   0          4m40s
 datahub-mysql-setup-job-b57kc                      0/1     Completed   0          4m7s
 elasticsearch-master-0                             1/1     Running     0          97m
-elasticsearch-master-1                             1/1     Running     0          97m
-elasticsearch-master-2                             1/1     Running     0          97m
 prerequisites-cp-schema-registry-cf79bfccf-kvjtv   2/2     Running     1          99m
 prerequisites-kafka-0                              1/1     Running     2          97m
 prerequisites-mysql-0                              1/1     Running     1          97m

charts/datahub/Chart.yaml

@@ -4,33 +4,33 @@ description: A Helm chart for LinkedIn DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.161
+version: 0.2.166
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 0.10.0
+appVersion: 0.10.3
 dependencies:
   - name: datahub-gms
-    version: 0.2.146
+    version: 0.2.148
     repository: file://./subcharts/datahub-gms
     condition: datahub-gms.enabled
   - name: datahub-frontend
-    version: 0.2.136
+    version: 0.2.137
     repository: file://./subcharts/datahub-frontend
     condition: datahub-frontend.enabled
   - name: datahub-mae-consumer
-    version: 0.2.142
+    version: 0.2.144
     repository: file://./subcharts/datahub-mae-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-mce-consumer
-    version: 0.2.144
+    version: 0.2.146
     repository: file://./subcharts/datahub-mce-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-ingestion-cron
-    version: 0.2.131
+    version: 0.2.132
     repository: file://./subcharts/datahub-ingestion-cron
     condition: datahub-ingestion-cron.enabled
   - name: acryl-datahub-actions
-    version: 0.2.134
+    version: 0.2.137
     repository: file://./subcharts/acryl-datahub-actions
     condition: acryl-datahub-actions.enabled
 maintainers:

charts/datahub/README.md

@@ -39,6 +39,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | datahubUpgrade.securityContext | object | `{}` | Container security context for datahubUpgrade jobs |
 | datahubUpgrade.podAnnotations | object | `{}` | Pod annotations for datahubUpgrade jobs |
 | datahubUpgrade.restoreIndices.resources | object | '{}' | Kube Resource definitions for the datahub upgrade job 'restore indices' |
+| datahubUpgrade.restoreIndices.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod |
 | elasticsearchSetupJob.enabled | bool | `true` | Enable setup job for elasicsearch |
 | elasticsearchSetupJob.image.repository | string | `"linkedin/datahub-elasticsearch-setup"` | Image repository for elasticsearchSetupJob |
 | elasticsearchSetupJob.image.tag | string | `"v0.10.0"` | Image repository for elasticsearchSetupJob |
@@ -47,6 +48,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | elasticsearchSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for elasticsearchSetupJob |
 | elasticsearchSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for elasticsearchSetupJob |
 | elasticsearchSetupJob.podAnnotations | object | `{}` | Pod annotations for elasticsearchSetupJob |
+| elasticsearchSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod |
 | kafkaSetupJob.enabled | bool | `true` | Enable setup job for kafka |
 | kafkaSetupJob.image.repository | string | `"linkedin/datahub-kafka-setup"` | Image repository for kafkaSetupJob |
 | kafkaSetupJob.image.tag | string | `"v0.10.0"` | Image repository for kafkaSetupJob |
@@ -55,6 +57,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | kafkaSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for kafkaSetupJob |
 | kafkaSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for kafkaSetupJob |
 | kafkaSetupJob.podAnnotations | object | `{}` | Pod annotations for kafkaSetupJob |
+| kafkaSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod |
 | mysqlSetupJob.enabled | bool | `false` | Enable setup job for mysql |
 | mysqlSetupJob.image.repository | string | `"acryldata/datahub-mysql-setup"` | Image repository for mysqlSetupJob |
 | mysqlSetupJob.image.tag | string | `"v0.10.0"` | Image repository for mysqlSetupJob |
@@ -63,6 +66,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | mysqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob |
 | mysqlSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for mysqlSetupJob |
 | mysqlSetupJob.podAnnotations | object | `{}` | Pod annotations for mysqlSetupJob |
+| mysqlSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod |
 | postgresqlSetupJob.enabled | bool | `false` | Enable setup job for postgresql |
 | postgresqlSetupJob.image.repository | string | `"acryldata/datahub-postgres-setup"` | Image repository for postgresqlSetupJob |
 | postgresqlSetupJob.image.tag | string | `"v0.10.0"` | Image repository for postgresqlSetupJob |
@@ -71,6 +75,8 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | postgresqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob |
 | postgresqlSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for mysqlSetupJob |
 | postgresqlSetupJob.podAnnotations | object | `{}` | Pod annotations for mysqlSetupJob |
+| postgresqlSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod |
+| datahubSystemUpdate.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod |
 | global.strict_mode | boolean | true | Enables validations in helm charts to ensure features work as expected. Recommended NOT TO CHANGE. |
 | global.datahub_standalone_consumers_enabled | boolean | true | Enable standalone consumers for kafka |
 | global.datahub_analytics_enabled | boolean | true | Enable datahub usage analytics |
@@ -89,7 +95,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | global.kafka.topics.metadata_change_log_versioned_topic_name | string | `"MetadataChangeLog_Versioned_v1"` | Kafka topic name for Versioned Metadata Change Log events |
 | global.kafka.topics.metadata_change_log_timeseries_topic_name | string | `"MetadataChangeLog_Timeseries_v1"` | Kafka topic name for Timeseries Metadata Change Log events |
 | global.kafka.topics.platform_event_topic_name | string | `"PlatformEvent_v1"` | Kafka topic name for Platform events |
-| global.kafka.schemaregistry.url | string | `"http://prerequisites-cp-schema-registry:8081"` | URL to kafka schema registry |
+| global.kafka.schemaregistry.url | string | `` | URL to kafka schema registry if using `KAFKA` type |
 | global.neo4j.host | string | `"prerequisites-neo4j:7474"` | Neo4j host address (with port) |
 | global.neo4j.uri | string | `"bolt://prerequisites-neo4j"` | Neo4j URI |
 | global.neo4j.username | string | `"neo4j"` | Neo4j user name |
@@ -123,6 +129,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | postgresqlSetupJob.password.secretKey | string | `"mysql-password"` | Secret key that contains the postgresqlSetupJob SQL password (overrides global value) |
 | postgresqlSetupJob.password.value | string | `"mysql-password"` | Alternative to using the secret above, uses raw string value for postgresqlSetupJob SQL login (overrides global value) |
 | acryl-datahub-actions.ingestionSecretFiles.name | string | `""` | Name of the k8s secret that holds any secret files (e.g., SSL certificates and private keys) that are used in your ingestion recipes. The keys in the secret will be mounted as individual files under `/etc/datahub/ingestion-secret-files` |
+| acryl-datahub-actions.ingestionSecretFiles.defaultMode | string | `""` | The permission mode for the volume that mounts k8s secret under `/etc/datahub/ingestion-secret-files`, default value is 0444 which allows read access by owner, group, and other users |
 | global.credentialsAndCertsSecrets.name | string | `""` | Name of the secret that holds SSL certificates (keystores, truststores) |
 | global.credentialsAndCertsSecrets.path | string | `"/mnt/certs"` | Path to mount the SSL certificates |
 | global.credentialsAndCertsSecrets.secureEnv | map | `{}` | Map of SSL config name and the corresponding value in the secret |
@@ -132,7 +139,7 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | global.elasticsearch.auth.password.secretRef | string | `""` | Secret that contains the elasticsearch password |
 | global.elasticsearch.auth.password.secretKey | string | `""` | Secret key that contains the elasticsearch password |
 | global.elasticsearch.auth.password.value | string | `""` | Alternative to using the secret above, uses raw string value instead |
-| global.kafka.schemaregistry.type | string | `"KAFKA"` | Type of schema registry (KAFKA or AWS_GLUE) |
+| global.kafka.schemaregistry.type | string | `"INTERNAL"` | Type of schema registry (INTERNAL, KAFKA, or AWS_GLUE) |
 | global.kafka.schemaregistry.glue.region | string | `""` | Region of the AWS Glue schema registry |
 | global.kafka.schemaregistry.glue.registry | string | `""` | Name of the AWS Glue schema registry |
 | datahub.metadata_service_authentication.enabled | bool | `false` | Whether Metadata Service Authentication is enabled. |

charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml

@@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.134
+version: 0.2.137
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.0.11

charts/datahub/subcharts/acryl-datahub-actions/README.md

@@ -12,6 +12,7 @@ Current chart version is `0.0.3`
 | exporters.jmx.enabled | boolean | false |  |
 | extraLabels | object | `{}` | Extra labels for deployment configuration |
 | extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` |
+| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) |
 | extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" |
 | extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" |
 | fullnameOverride | string | `"acryl-datahub-actions"` |  |
@@ -41,3 +42,5 @@ Current chart version is `0.0.3`
 | global.datahub.metadata_service_authentication.systemClientId | string | `"__datahub_system"` | The internal system id that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. |
 | global.datahub.metadata_service_authentication.systemClientSecret.secretRef | string | `nil` | The reference to a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. |
 | global.datahub.metadata_service_authentication.systemClientSecret.secretKey | string | `nil` | The key of a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. |
+| ingestionSecretFiles.name | string | `""` | Name of the k8s secret that holds any secret files (e.g., SSL certificates and private keys) that are used in your ingestion recipes. The keys in the secret will be mounted as individual files under `/etc/datahub/ingestion-secret-files` |
+| ingestionSecretFiles.defaultMode | string | `""` | The permission mode for the volume that mounts k8s secret under `/etc/datahub/ingestion-secret-files`, default value is 0444 which allows read access by owner, group, and other users |

charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml

@@ -41,7 +41,7 @@ spec:
       {{- with .Values.ingestionSecretFiles }}
         - name: ingestion-secret-files
           secret:
-            defaultMode: 0444
+            defaultMode: {{ .defaultMode }}
             secretName: {{ .name }}
       {{- end }}
       {{- if .Values.extraVolumes }}
@@ -73,8 +73,13 @@ spec:
               value: "{{ .Values.global.datahub.gms.port }}"
             - name: KAFKA_BOOTSTRAP_SERVER
               value: "{{ .Values.global.kafka.bootstrap.server }}"
+            {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }}
+            - name: SCHEMA_REGISTRY_URL
+              value: {{ printf "http://%s-%s:%s/schema-registry/api/" .Release.Name "datahub-gms" .Values.global.datahub.gms.port }}
+            {{- else if eq .Values.global.kafka.schemaregistry.type "KAFKA" }}
             - name: SCHEMA_REGISTRY_URL
               value: "{{ .Values.global.kafka.schemaregistry.url }}"
+            {{- end }}
             - name: KAFKA_AUTO_OFFSET_POLICY
               value: "{{ .Values.actions.kafkaAutoOffsetPolicy }}"
             {{- if .Values.global.springKafkaConfigurationOverrides }}
@@ -121,7 +126,7 @@ spec:
                   name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }}
                   key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }}
             {{- end }}
-           {{- if .Values.extraEnvs }}
+          {{- if .Values.extraEnvs }}
             {{ toYaml .Values.extraEnvs | nindent 12 }}
           {{- end }}
           volumeMounts:
@@ -139,7 +144,9 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
-
+        {{- with .Values.extraSidecars }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/datahub/subcharts/acryl-datahub-actions/values.yaml

@@ -50,6 +50,12 @@ extraVolumeMounts: []
 
 extraInitContainers: []
 
+# Add extra sidecar containers to deployment pod
+extraSidecars: []
+  # - name: my-image-name
+  #   image: my-image
+  #   imagePullPolicy: Always
+
 resources: {}
 
 nodeSelector: {}
@@ -61,6 +67,11 @@ affinity: {}
 actions:
   kafkaAutoOffsetPolicy: "latest"
 
+# mount the k8s secret as a volume in the container, each key name is mounted as a file on the mount path /etc/datahub/ingestion-secret-files
+# ingestionSecretFiles:
+#   name: ${K8S_SECRET_NAME}
+#   defaultMode: "0444"
+
 global:
 
   kafka:

charts/datahub/subcharts/datahub-frontend/Chart.yaml

@@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.136
+version: 0.2.137
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.10.0

charts/datahub/subcharts/datahub-frontend/README.md

@@ -14,6 +14,7 @@ Current chart version is `0.2.0`
 | exporters.jmx.enabled | boolean | false |  |
 | extraLabels | object | `{}` | Extra labels for deployment configuration |
 | extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` |
+| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) |
 | extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" |
 | extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" |
 | fullnameOverride | string | `"datahub-frontend"` |  |

charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml

@@ -219,7 +219,9 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
-
+        {{- with .Values.extraSidecars }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/datahub/subcharts/datahub-frontend/values.yaml

@@ -110,6 +110,12 @@ extraVolumeMounts: []
 
 extraInitContainers: []
 
+# Add extra sidecar containers to deployment pod
+extraSidecars: []
+  # - name: my-image-name
+  #   image: my-image
+  #   imagePullPolicy: Always
+
 lifecycle: {}
   # To add a new user to datahub in JAAS config without mounting the user.props file
   # postStart:

charts/datahub/subcharts/datahub-gms/Chart.yaml

@@ -12,7 +12,7 @@ description: A Helm chart for LinkedIn DataHub's datahub-gms component
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.146
+version: 0.2.148
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.10.0

charts/datahub/subcharts/datahub-gms/README.md

@@ -11,6 +11,7 @@ Current chart version is `0.2.0`
 | affinity | object | `{}` |  |
 | extraLabels | object | `{}` | Extra labels for deployment configuration |
 | extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` |
+| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) |
 | extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" |
 | extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" |
 | fullnameOverride | string | `"datahub-gms-deployment"` |  |

charts/datahub/subcharts/datahub-gms/templates/configmap.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.global.elasticsearch.search.custom.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s-%s" .Release.Name "search-custom" }}
+  labels:
+    {{- include "datahub-gms.labels" . | nindent 4 }}
+    {{- range $key, $val := .Values.extraLabels }}
+    {{ $key }}: {{ $val | quote }}
+    {{- end }}
+data:
+  search_config.yml: |
+    {{- toYaml .Values.global.elasticsearch.search.custom.config | nindent 4 }}
+{{- end }}