From f55a2fb6d0b0635b51a59a75215971bcc563af40 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Fri, 24 Mar 2023 11:30:41 +0000 Subject: [PATCH 1/8] Release Charts for DataHub v0.10.1 (#288) * fix(secrets): Make secret creation optional Makes datahub-auth-secrets creation dependant on whether metadata service authentication has been enabled or not. * Bump chart version & default datahub version --- charts/datahub/Chart.yaml | 4 ++-- charts/datahub/templates/datahub-auth-secrets.yml | 2 ++ charts/datahub/values.yaml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index fb57479d6..3af021249 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,10 +4,10 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.160 +version: 0.2.161 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 0.10.0 +appVersion: 0.10.1 dependencies: - name: datahub-gms version: 0.2.146 diff --git a/charts/datahub/templates/datahub-auth-secrets.yml b/charts/datahub/templates/datahub-auth-secrets.yml index fcc2fe1ac..1aa8ad58a 100644 --- a/charts/datahub/templates/datahub-auth-secrets.yml +++ b/charts/datahub/templates/datahub-auth-secrets.yml @@ -1,3 +1,4 @@ +{{- if .Values.global.datahub.metadata_service_authentication.enabled -}} {{- $secret := lookup "v1" "Secret" .Release.Namespace "datahub-auth-secrets" -}} {{- $data := $secret.data | default dict -}} {{- with .Values.global.datahub.metadata_service_authentication.provisionSecrets }} @@ -21,3 +22,4 @@ data: {{- end }} {{- end -}} +{{- end -}} diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index f201a32bf..238cd6410 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -344,7 +344,7 @@ global: # value: password datahub: - version: v0.10.0 + version: v0.10.1 gms: port: "8080" nodePort: "30001" From f2269e3649c4e131f62a4c0e0bedbe4764628a54 Mon Sep 17 00:00:00 2001 From: Upendra Rao Vedullapalli Date: Fri, 24 Mar 2023 12:33:30 +0100 Subject: [PATCH 2/8] fix: secretkey for systemClientSecret in values.yaml file (#280) * fix: systemClientSecret key in values.yaml Updated systemClientSecret with proper secretKey from the datahub-auth-secrets file * fix: Updated Chart version --------- Co-authored-by: Upendra Vedullapalli --- charts/datahub/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 238cd6410..8f704d2c8 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -382,7 +382,7 @@ global: systemClientId: "__datahub_system" systemClientSecret: secretRef: "datahub-auth-secrets" - secretKey: "token_service_signing_key" + secretKey: "system_client_secret" tokenService: signingKey: secretRef: "datahub-auth-secrets" From a4d7cb9a8729f1229d7cd5bc5af80473547aeffb Mon Sep 17 00:00:00 2001 From: Kevin Meyer Date: Mon, 3 Apr 2023 21:42:01 -0400 Subject: [PATCH 3/8] feat(PFP-102): Make Elasticsearch use 1 master node by default (#283) Our dependencies chart is setup to use 3 ES nodes in 3 AZ-equivalents, but is broadly intended for use as a test harness which often runs in single-node events. So let's fix this and have it be single-node by default --- README.md | 4 ---- charts/prerequisites/Chart.yaml | 2 +- charts/prerequisites/values.yaml | 11 +++++++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index fc8072831..46355ed73 100644 --- a/README.md +++ b/README.md @@ -76,8 +76,6 @@ You should get a result similar to below. ``` NAME READY STATUS RESTARTS AGE elasticsearch-master-0 1/1 Running 0 62m -elasticsearch-master-1 1/1 Running 0 62m -elasticsearch-master-2 1/1 Running 0 62m prerequisites-cp-schema-registry-cf79bfccf-kvjtv 2/2 Running 1 63m prerequisites-kafka-0 1/1 Running 2 62m prerequisites-mysql-0 1/1 Running 1 62m @@ -108,8 +106,6 @@ datahub-elasticsearch-setup-job-8dz6b 0/1 Completed 0 datahub-kafka-setup-job-6blcj 0/1 Completed 0 4m40s datahub-mysql-setup-job-b57kc 0/1 Completed 0 4m7s elasticsearch-master-0 1/1 Running 0 97m -elasticsearch-master-1 1/1 Running 0 97m -elasticsearch-master-2 1/1 Running 0 97m prerequisites-cp-schema-registry-cf79bfccf-kvjtv 2/2 Running 1 99m prerequisites-kafka-0 1/1 Running 2 97m prerequisites-mysql-0 1/1 Running 1 97m diff --git a/charts/prerequisites/Chart.yaml b/charts/prerequisites/Chart.yaml index aa9e57e2e..1fbf1443e 100644 --- a/charts/prerequisites/Chart.yaml +++ b/charts/prerequisites/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for packages that Datahub depends on type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.0.14 +version: 0.0.15 dependencies: - name: elasticsearch version: 7.17.3 diff --git a/charts/prerequisites/values.yaml b/charts/prerequisites/values.yaml index c5c0dfb6d..1968f5481 100644 --- a/charts/prerequisites/values.yaml +++ b/charts/prerequisites/values.yaml @@ -2,14 +2,17 @@ # Copy this file and update to the configuration of choice elasticsearch: enabled: true # set this to false, if you want to provide your own ES instance. - replicas: 3 + + # If you're running in production, set this to 3 and comment out antiAffinity below + # Or alternatively if you're running production, bring your own ElasticSearch + replicas: 1 minimumMasterNodes: 1 # Set replicas to 1 and uncomment this to allow the instance to be scheduled on # a master node when deploying on a single node Minikube / Kind / etc cluster. - # antiAffinity: "soft" + antiAffinity: "soft" - # # If your running a single replica cluster add the following helm value - # clusterHealthCheckParams: "wait_for_status=yellow&timeout=1s" + # # If you are running a multi-replica cluster, comment this out + clusterHealthCheckParams: "wait_for_status=yellow&timeout=1s" # # Shrink default JVM heap. esJavaOpts: "-Xmx384m -Xms384m" From 3133f567d3e351f015c82b9404cc2552240ba1f6 Mon Sep 17 00:00:00 2001 From: Tony Ouyang <43738225+TonyOuyangGit@users.noreply.github.com> Date: Wed, 5 Apr 2023 13:59:26 -0700 Subject: [PATCH 4/8] feat: change k8s secret mount default mode to accept value override (#289) * update deployment yaml file * bump chart version * add missing chart version update * address comments to add descriptions in README and values.yaml * address comment to remove default value of defaultMode --- charts/datahub/Chart.yaml | 4 ++-- charts/datahub/README.md | 1 + charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml | 2 +- charts/datahub/subcharts/acryl-datahub-actions/README.md | 2 ++ .../acryl-datahub-actions/templates/deployment.yaml | 2 +- charts/datahub/subcharts/acryl-datahub-actions/values.yaml | 5 +++++ charts/datahub/values.yaml | 4 ++++ 7 files changed, 16 insertions(+), 4 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 3af021249..e2c4f6677 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.161 +version: 0.2.162 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.10.1 @@ -30,7 +30,7 @@ dependencies: repository: file://./subcharts/datahub-ingestion-cron condition: datahub-ingestion-cron.enabled - name: acryl-datahub-actions - version: 0.2.134 + version: 0.2.135 repository: file://./subcharts/acryl-datahub-actions condition: acryl-datahub-actions.enabled maintainers: diff --git a/charts/datahub/README.md b/charts/datahub/README.md index a4f0a26ed..668208481 100644 --- a/charts/datahub/README.md +++ b/charts/datahub/README.md @@ -123,6 +123,7 @@ helm install datahub datahub/datahub --values <> | postgresqlSetupJob.password.secretKey | string | `"mysql-password"` | Secret key that contains the postgresqlSetupJob SQL password (overrides global value) | | postgresqlSetupJob.password.value | string | `"mysql-password"` | Alternative to using the secret above, uses raw string value for postgresqlSetupJob SQL login (overrides global value) | | acryl-datahub-actions.ingestionSecretFiles.name | string | `""` | Name of the k8s secret that holds any secret files (e.g., SSL certificates and private keys) that are used in your ingestion recipes. The keys in the secret will be mounted as individual files under `/etc/datahub/ingestion-secret-files` | +| acryl-datahub-actions.ingestionSecretFiles.defaultMode | string | `""` | The permission mode for the volume that mounts k8s secret under `/etc/datahub/ingestion-secret-files`, default value is 0444 which allows read access by owner, group, and other users | | global.credentialsAndCertsSecrets.name | string | `""` | Name of the secret that holds SSL certificates (keystores, truststores) | | global.credentialsAndCertsSecrets.path | string | `"/mnt/certs"` | Path to mount the SSL certificates | | global.credentialsAndCertsSecrets.secureEnv | map | `{}` | Map of SSL config name and the corresponding value in the secret | diff --git a/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml b/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml index 489b63082..ddbf3abad 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.134 +version: 0.2.135 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.0.11 diff --git a/charts/datahub/subcharts/acryl-datahub-actions/README.md b/charts/datahub/subcharts/acryl-datahub-actions/README.md index a8f5ede0a..e4cf24467 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/README.md +++ b/charts/datahub/subcharts/acryl-datahub-actions/README.md @@ -41,3 +41,5 @@ Current chart version is `0.0.3` | global.datahub.metadata_service_authentication.systemClientId | string | `"__datahub_system"` | The internal system id that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | | global.datahub.metadata_service_authentication.systemClientSecret.secretRef | string | `nil` | The reference to a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | | global.datahub.metadata_service_authentication.systemClientSecret.secretKey | string | `nil` | The key of a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | +| ingestionSecretFiles.name | string | `""` | Name of the k8s secret that holds any secret files (e.g., SSL certificates and private keys) that are used in your ingestion recipes. The keys in the secret will be mounted as individual files under `/etc/datahub/ingestion-secret-files` | +| ingestionSecretFiles.defaultMode | string | `""` | The permission mode for the volume that mounts k8s secret under `/etc/datahub/ingestion-secret-files`, default value is 0444 which allows read access by owner, group, and other users | diff --git a/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml b/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml index 365cfae18..47427a616 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml @@ -41,7 +41,7 @@ spec: {{- with .Values.ingestionSecretFiles }} - name: ingestion-secret-files secret: - defaultMode: 0444 + defaultMode: {{ .defaultMode }} secretName: {{ .name }} {{- end }} {{- if .Values.extraVolumes }} diff --git a/charts/datahub/subcharts/acryl-datahub-actions/values.yaml b/charts/datahub/subcharts/acryl-datahub-actions/values.yaml index 7877c344d..e5a0ceb56 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/values.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/values.yaml @@ -61,6 +61,11 @@ affinity: {} actions: kafkaAutoOffsetPolicy: "latest" +# mount the k8s secret as a volume in the container, each key name is mounted as a file on the mount path /etc/datahub/ingestion-secret-files +# ingestionSecretFiles: +# name: ${K8S_SECRET_NAME} +# defaultMode: "0444" + global: kafka: diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 8f704d2c8..26a65dbcf 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -39,6 +39,10 @@ acryl-datahub-actions: image: repository: acryldata/datahub-actions tag: "v0.0.11" + # mount the k8s secret as a volume in the container, each key name is mounted as a file on the mount path /etc/datahub/ingestion-secret-files + # ingestionSecretFiles: + # name: ${K8S_SECRET_NAME} + # defaultMode: "0444" resources: limits: memory: 512Mi From e9fecf7fe203a7eb7bf41821afe78c1c90ce83eb Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Fri, 14 Apr 2023 16:14:36 -0500 Subject: [PATCH 5/8] Update helm charts for datahub 0.10.2 release (#298) Co-authored-by: Indy Prentice --- charts/datahub/Chart.yaml | 4 ++-- charts/datahub/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index e2c4f6677..b91a97923 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,10 +4,10 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.162 +version: 0.2.163 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 0.10.1 +appVersion: 0.10.2 dependencies: - name: datahub-gms version: 0.2.146 diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 26a65dbcf..7bfbff200 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -348,7 +348,7 @@ global: # value: password datahub: - version: v0.10.1 + version: v0.10.2 gms: port: "8080" nodePort: "30001" From 09a9063443af0d8d56727864dd52b280c11b568a Mon Sep 17 00:00:00 2001 From: Matthijs van der Loos <33864538+matthijsvanderloos@users.noreply.github.com> Date: Thu, 20 Apr 2023 16:50:01 +0200 Subject: [PATCH 6/8] feat: enable sidecar configuration (#276) * feat: enable sidecar configuration --- charts/datahub/Chart.yaml | 14 +++---- charts/datahub/README.md | 6 +++ .../acryl-datahub-actions/Chart.yaml | 2 +- .../subcharts/acryl-datahub-actions/README.md | 1 + .../templates/deployment.yaml | 6 ++- .../acryl-datahub-actions/values.yaml | 6 +++ .../subcharts/datahub-frontend/Chart.yaml | 2 +- .../subcharts/datahub-frontend/README.md | 1 + .../templates/deployment.yaml | 4 +- .../subcharts/datahub-frontend/values.yaml | 6 +++ .../datahub/subcharts/datahub-gms/Chart.yaml | 2 +- .../datahub/subcharts/datahub-gms/README.md | 1 + .../datahub-gms/templates/deployment.yaml | 4 +- .../datahub/subcharts/datahub-gms/values.yaml | 6 +++ .../datahub-ingestion-cron/Chart.yaml | 2 +- .../datahub-ingestion-cron/README.md | 1 + .../templates/cron.yaml | 9 ++-- .../datahub-ingestion-cron/values.yaml | 14 +++++++ .../subcharts/datahub-mae-consumer/Chart.yaml | 2 +- .../subcharts/datahub-mae-consumer/README.md | 1 + .../templates/deployment.yaml | 4 +- .../datahub-mae-consumer/values.yaml | 6 +++ .../subcharts/datahub-mce-consumer/Chart.yaml | 2 +- .../subcharts/datahub-mce-consumer/README.md | 1 + .../templates/deployment.yaml | 4 +- .../datahub-mce-consumer/values.yaml | 6 +++ .../datahub-cleanup-job-template.yml | 3 ++ .../datahub-nocode-migration-job.yml | 3 ++ .../datahub-restore-indices-job-template.yml | 3 ++ .../datahub-system-update-job.yml | 3 ++ .../templates/elasticsearch-setup-job.yml | 3 ++ charts/datahub/templates/kafka-setup-job.yml | 3 ++ charts/datahub/templates/mysql-setup-job.yml | 3 ++ .../templates/postgresql-setup-job.yml | 3 ++ charts/datahub/values.yaml | 41 +++++++++++++++++++ 35 files changed, 156 insertions(+), 22 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index b91a97923..7dc325c75 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,33 +4,33 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.163 +version: 0.2.164 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.10.2 dependencies: - name: datahub-gms - version: 0.2.146 + version: 0.2.147 repository: file://./subcharts/datahub-gms condition: datahub-gms.enabled - name: datahub-frontend - version: 0.2.136 + version: 0.2.137 repository: file://./subcharts/datahub-frontend condition: datahub-frontend.enabled - name: datahub-mae-consumer - version: 0.2.142 + version: 0.2.143 repository: file://./subcharts/datahub-mae-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-mce-consumer - version: 0.2.144 + version: 0.2.145 repository: file://./subcharts/datahub-mce-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-ingestion-cron - version: 0.2.131 + version: 0.2.132 repository: file://./subcharts/datahub-ingestion-cron condition: datahub-ingestion-cron.enabled - name: acryl-datahub-actions - version: 0.2.135 + version: 0.2.136 repository: file://./subcharts/acryl-datahub-actions condition: acryl-datahub-actions.enabled maintainers: diff --git a/charts/datahub/README.md b/charts/datahub/README.md index 668208481..8a3e16bda 100644 --- a/charts/datahub/README.md +++ b/charts/datahub/README.md @@ -39,6 +39,7 @@ helm install datahub datahub/datahub --values <> | datahubUpgrade.securityContext | object | `{}` | Container security context for datahubUpgrade jobs | | datahubUpgrade.podAnnotations | object | `{}` | Pod annotations for datahubUpgrade jobs | | datahubUpgrade.restoreIndices.resources | object | '{}' | Kube Resource definitions for the datahub upgrade job 'restore indices' | +| datahubUpgrade.restoreIndices.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | elasticsearchSetupJob.enabled | bool | `true` | Enable setup job for elasicsearch | | elasticsearchSetupJob.image.repository | string | `"linkedin/datahub-elasticsearch-setup"` | Image repository for elasticsearchSetupJob | | elasticsearchSetupJob.image.tag | string | `"v0.10.0"` | Image repository for elasticsearchSetupJob | @@ -47,6 +48,7 @@ helm install datahub datahub/datahub --values <> | elasticsearchSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for elasticsearchSetupJob | | elasticsearchSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for elasticsearchSetupJob | | elasticsearchSetupJob.podAnnotations | object | `{}` | Pod annotations for elasticsearchSetupJob | +| elasticsearchSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | kafkaSetupJob.enabled | bool | `true` | Enable setup job for kafka | | kafkaSetupJob.image.repository | string | `"linkedin/datahub-kafka-setup"` | Image repository for kafkaSetupJob | | kafkaSetupJob.image.tag | string | `"v0.10.0"` | Image repository for kafkaSetupJob | @@ -55,6 +57,7 @@ helm install datahub datahub/datahub --values <> | kafkaSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for kafkaSetupJob | | kafkaSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for kafkaSetupJob | | kafkaSetupJob.podAnnotations | object | `{}` | Pod annotations for kafkaSetupJob | +| kafkaSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | mysqlSetupJob.enabled | bool | `false` | Enable setup job for mysql | | mysqlSetupJob.image.repository | string | `"acryldata/datahub-mysql-setup"` | Image repository for mysqlSetupJob | | mysqlSetupJob.image.tag | string | `"v0.10.0"` | Image repository for mysqlSetupJob | @@ -63,6 +66,7 @@ helm install datahub datahub/datahub --values <> | mysqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob | | mysqlSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for mysqlSetupJob | | mysqlSetupJob.podAnnotations | object | `{}` | Pod annotations for mysqlSetupJob | +| mysqlSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | postgresqlSetupJob.enabled | bool | `false` | Enable setup job for postgresql | | postgresqlSetupJob.image.repository | string | `"acryldata/datahub-postgres-setup"` | Image repository for postgresqlSetupJob | | postgresqlSetupJob.image.tag | string | `"v0.10.0"` | Image repository for postgresqlSetupJob | @@ -71,6 +75,8 @@ helm install datahub datahub/datahub --values <> | postgresqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob | | postgresqlSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for mysqlSetupJob | | postgresqlSetupJob.podAnnotations | object | `{}` | Pod annotations for mysqlSetupJob | +| postgresqlSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | +| datahubSystemUpdate.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | global.strict_mode | boolean | true | Enables validations in helm charts to ensure features work as expected. Recommended NOT TO CHANGE. | | global.datahub_standalone_consumers_enabled | boolean | true | Enable standalone consumers for kafka | | global.datahub_analytics_enabled | boolean | true | Enable datahub usage analytics | diff --git a/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml b/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml index ddbf3abad..0c0e1a588 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.135 +version: 0.2.136 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.0.11 diff --git a/charts/datahub/subcharts/acryl-datahub-actions/README.md b/charts/datahub/subcharts/acryl-datahub-actions/README.md index e4cf24467..e08afdbd0 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/README.md +++ b/charts/datahub/subcharts/acryl-datahub-actions/README.md @@ -12,6 +12,7 @@ Current chart version is `0.0.3` | exporters.jmx.enabled | boolean | false | | | extraLabels | object | `{}` | Extra labels for deployment configuration | | extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | | extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | | extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | | fullnameOverride | string | `"acryl-datahub-actions"` | | diff --git a/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml b/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml index 47427a616..aa1f79981 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml @@ -121,7 +121,7 @@ spec: name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} {{- end }} - {{- if .Values.extraEnvs }} + {{- if .Values.extraEnvs }} {{ toYaml .Values.extraEnvs | nindent 12 }} {{- end }} volumeMounts: @@ -139,7 +139,9 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - + {{- with .Values.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/datahub/subcharts/acryl-datahub-actions/values.yaml b/charts/datahub/subcharts/acryl-datahub-actions/values.yaml index e5a0ceb56..32fb3b282 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/values.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/values.yaml @@ -50,6 +50,12 @@ extraVolumeMounts: [] extraInitContainers: [] +# Add extra sidecar containers to deployment pod +extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always + resources: {} nodeSelector: {} diff --git a/charts/datahub/subcharts/datahub-frontend/Chart.yaml b/charts/datahub/subcharts/datahub-frontend/Chart.yaml index c69a9a274..96b0e0364 100644 --- a/charts/datahub/subcharts/datahub-frontend/Chart.yaml +++ b/charts/datahub/subcharts/datahub-frontend/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.136 +version: 0.2.137 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.10.0 diff --git a/charts/datahub/subcharts/datahub-frontend/README.md b/charts/datahub/subcharts/datahub-frontend/README.md index 366860f09..7c14edbe7 100644 --- a/charts/datahub/subcharts/datahub-frontend/README.md +++ b/charts/datahub/subcharts/datahub-frontend/README.md @@ -14,6 +14,7 @@ Current chart version is `0.2.0` | exporters.jmx.enabled | boolean | false | | | extraLabels | object | `{}` | Extra labels for deployment configuration | | extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | | extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | | extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | | fullnameOverride | string | `"datahub-frontend"` | | diff --git a/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml b/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml index a850cc4b3..8fa84c1d0 100644 --- a/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml @@ -219,7 +219,9 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - + {{- with .Values.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/datahub/subcharts/datahub-frontend/values.yaml b/charts/datahub/subcharts/datahub-frontend/values.yaml index d4f6bb8c2..77ac4075a 100644 --- a/charts/datahub/subcharts/datahub-frontend/values.yaml +++ b/charts/datahub/subcharts/datahub-frontend/values.yaml @@ -110,6 +110,12 @@ extraVolumeMounts: [] extraInitContainers: [] +# Add extra sidecar containers to deployment pod +extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always + lifecycle: {} # To add a new user to datahub in JAAS config without mounting the user.props file # postStart: diff --git a/charts/datahub/subcharts/datahub-gms/Chart.yaml b/charts/datahub/subcharts/datahub-gms/Chart.yaml index fc8b99868..ae513efc1 100644 --- a/charts/datahub/subcharts/datahub-gms/Chart.yaml +++ b/charts/datahub/subcharts/datahub-gms/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for LinkedIn DataHub's datahub-gms component type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.146 +version: 0.2.147 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.10.0 diff --git a/charts/datahub/subcharts/datahub-gms/README.md b/charts/datahub/subcharts/datahub-gms/README.md index 3113e2ed8..24df024ae 100644 --- a/charts/datahub/subcharts/datahub-gms/README.md +++ b/charts/datahub/subcharts/datahub-gms/README.md @@ -11,6 +11,7 @@ Current chart version is `0.2.0` | affinity | object | `{}` | | | extraLabels | object | `{}` | Extra labels for deployment configuration | | extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | | extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | | extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | | fullnameOverride | string | `"datahub-gms-deployment"` | | diff --git a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml index 574339fa3..4a9be56b7 100644 --- a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml @@ -336,7 +336,9 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - + {{- with .Values.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/datahub/subcharts/datahub-gms/values.yaml b/charts/datahub/subcharts/datahub-gms/values.yaml index b50e172a4..fc06079fd 100644 --- a/charts/datahub/subcharts/datahub-gms/values.yaml +++ b/charts/datahub/subcharts/datahub-gms/values.yaml @@ -92,6 +92,12 @@ extraVolumeMounts: [] extraInitContainers: [] +# Add extra sidecar containers to deployment pod +extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml index ce8b3d707..f39f08f72 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.131 +version: 0.2.132 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.10.0 diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/README.md b/charts/datahub/subcharts/datahub-ingestion-cron/README.md index 435948462..df7feddda 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/README.md +++ b/charts/datahub/subcharts/datahub-ingestion-cron/README.md @@ -27,3 +27,4 @@ A Helm chart for datahub's metadata-ingestion framework with kerberos authentica | crons.extraInitContainers | object | `{}` | Init containers to add to the cronjob container | | crons.serviceAccountName | string | | Service account name used for the cronjob container | | crons.podAnnotations | object | `{}` | Annotations to add to the pods | +| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml index 872dd6960..be3ae29b9 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml @@ -11,11 +11,11 @@ spec: schedule: {{ default "0 0 * * *" .schedule | quote}} jobTemplate: spec: - template: + template: {{- with $val.podAnnotations }} metadata: annotations: - {{- toYaml . | nindent 12 }} + {{- toYaml . | nindent 12 }} {{- end }} spec: {{- with $.Values.imagePullSecrets }} @@ -68,5 +68,8 @@ spec: {{- if .extraVolumes }} {{- toYaml .extraVolumes | nindent 12 }} {{- end }} + {{- if .extraSidecars }} + {{- toYaml .extraSidecars | nindent 10 }} + {{- end }} --- -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml index 1d2e2b0f3..157960488 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml @@ -51,6 +51,13 @@ crons: {} ## #extraInitContainers: {} + ## Add extra sidecar containers to job pod + ## + # extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always + ## If you want to specify your own service account, set its name like so. ## #serviceAccountName: "my-cron-service" @@ -58,6 +65,13 @@ crons: {} ## Add your own pod annotations. ## #podAnnotations: {} + +# Add extra sidecar containers to deployment pod +extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always + global: datahub: version: head \ No newline at end of file diff --git a/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml index 00237aa88..fd7d46e1f 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml +++ b/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.142 +version: 0.2.143 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.10.0 diff --git a/charts/datahub/subcharts/datahub-mae-consumer/README.md b/charts/datahub/subcharts/datahub-mae-consumer/README.md index 41f7d7b73..11b733782 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/README.md +++ b/charts/datahub/subcharts/datahub-mae-consumer/README.md @@ -12,6 +12,7 @@ Current chart version is `0.2.0` | exporters.jmx.enabled | boolean | false | | | extraLabels | object | `{}` | Extra labels for deployment configuration | | extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | | extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | | extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | | fullnameOverride | string | `"datahub-mae-consumer"` | | diff --git a/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml index 18d804265..4b6e1ad3d 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml @@ -230,7 +230,9 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - + {{- with .Values.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/datahub/subcharts/datahub-mae-consumer/values.yaml b/charts/datahub/subcharts/datahub-mae-consumer/values.yaml index 3118918b8..2a2b91ea9 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/values.yaml +++ b/charts/datahub/subcharts/datahub-mae-consumer/values.yaml @@ -84,6 +84,12 @@ extraVolumeMounts: [] extraInitContainers: [] +# Add extra sidecar containers to deployment pod +extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little diff --git a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml index 9963fdc12..241d0827d 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml +++ b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.144 +version: 0.2.145 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.10.0 diff --git a/charts/datahub/subcharts/datahub-mce-consumer/README.md b/charts/datahub/subcharts/datahub-mce-consumer/README.md index 6be06395c..5f20dd759 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/README.md +++ b/charts/datahub/subcharts/datahub-mce-consumer/README.md @@ -12,6 +12,7 @@ Current chart version is `0.2.0` | exporters.jmx.enabled | boolean | false | | | extraLabels | object | `{}` | Extra labels for deployment configuration | | extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | | extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | | extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | | fullnameOverride | string | `""` | | diff --git a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml index 5dc696489..1fea2719a 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml @@ -235,7 +235,9 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - + {{- with .Values.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/datahub/subcharts/datahub-mce-consumer/values.yaml b/charts/datahub/subcharts/datahub-mce-consumer/values.yaml index 79aa112b2..75769726a 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/values.yaml +++ b/charts/datahub/subcharts/datahub-mce-consumer/values.yaml @@ -85,6 +85,12 @@ extraVolumeMounts: [] extraInitContainers: [] +# Add extra sidecar containers to deployment pod +extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little diff --git a/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml b/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml index c8c6d106a..2d1524595 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml @@ -77,6 +77,9 @@ spec: requests: cpu: 300m memory: 256Mi + {{- with .Values.datahubUpgrade.cleanupJob.extraSidecars }} + {{- toYaml . | nindent 12 }} + {{- end }} {{- with .Values.datahubUpgrade.nodeSelector }} nodeSelector: {{- toYaml . | nindent 12 }} diff --git a/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml index 4ea5b50e5..afe181e39 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml @@ -93,6 +93,9 @@ spec: requests: cpu: 300m memory: 256Mi + {{- with .Values.datahubUpgrade.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.datahubUpgrade.nodeSelector }} nodeSelector: {{- toYaml . | nindent 12 }} diff --git a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml index bb9fae651..392438dd6 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml @@ -88,6 +88,9 @@ spec: {{- end }} resources: {{- toYaml .Values.datahubUpgrade.restoreIndices.resources | nindent 16}} + {{- with .Values.datahubUpgrade.restoreIndices.extraSidecars }} + {{- toYaml . | nindent 12 }} + {{- end }} {{- with .Values.datahubUpgrade.nodeSelector }} nodeSelector: {{- toYaml . | nindent 12 }} diff --git a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml index 40e3eb046..29786df8e 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml @@ -124,6 +124,9 @@ spec: {{- end }} resources: {{- toYaml .Values.datahubSystemUpdate.resources | nindent 12 }} + {{- with .Values.datahubSystemUpdate.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.datahubSystemUpdate.nodeSelector }} nodeSelector: {{- toYaml . | nindent 12 }} diff --git a/charts/datahub/templates/elasticsearch-setup-job.yml b/charts/datahub/templates/elasticsearch-setup-job.yml index 9ba5b0ba2..d9382a4b5 100644 --- a/charts/datahub/templates/elasticsearch-setup-job.yml +++ b/charts/datahub/templates/elasticsearch-setup-job.yml @@ -95,6 +95,9 @@ spec: {{- end }} resources: {{- toYaml .Values.elasticsearchSetupJob.resources | nindent 12 }} + {{- with .Values.elasticsearchSetupJob.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.elasticsearchSetupJob.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/datahub/templates/kafka-setup-job.yml b/charts/datahub/templates/kafka-setup-job.yml index 2fb4797d6..86bc43980 100644 --- a/charts/datahub/templates/kafka-setup-job.yml +++ b/charts/datahub/templates/kafka-setup-job.yml @@ -123,6 +123,9 @@ spec: {{- end }} resources: {{- toYaml .Values.kafkaSetupJob.resources | nindent 12 }} + {{- with .Values.kafkaSetupJob.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.kafkaSetupJob.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/datahub/templates/mysql-setup-job.yml b/charts/datahub/templates/mysql-setup-job.yml index 38c92e0a6..c41a1f611 100644 --- a/charts/datahub/templates/mysql-setup-job.yml +++ b/charts/datahub/templates/mysql-setup-job.yml @@ -80,6 +80,9 @@ spec: {{- end }} resources: {{- toYaml .Values.mysqlSetupJob.resources | nindent 12 }} + {{- with .Values.mysqlSetupJob.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.mysqlSetupJob.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/datahub/templates/postgresql-setup-job.yml b/charts/datahub/templates/postgresql-setup-job.yml index c03161971..f3a5a421f 100644 --- a/charts/datahub/templates/postgresql-setup-job.yml +++ b/charts/datahub/templates/postgresql-setup-job.yml @@ -80,6 +80,9 @@ spec: {{- end }} resources: {{- toYaml .Values.postgresqlSetupJob.resources | nindent 12 }} + {{- with .Values.postgresqlSetupJob.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.postgresqlSetupJob.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 7bfbff200..02b808b74 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -95,6 +95,11 @@ elasticsearchSetupJob: securityContext: runAsUser: 1000 podAnnotations: {} + # Add extra sidecar containers to job pod + extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always kafkaSetupJob: enabled: true @@ -113,6 +118,11 @@ kafkaSetupJob: securityContext: runAsUser: 1000 podAnnotations: {} + # Add extra sidecar containers to job pod + extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always mysqlSetupJob: enabled: true @@ -136,6 +146,11 @@ mysqlSetupJob: # password: # secretRef: mysqlSetupJob-secret # secretKey: mysqlSetupJob-password + # Add extra sidecar containers to job pod + extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always postgresqlSetupJob: enabled: false @@ -159,6 +174,11 @@ postgresqlSetupJob: # password: # secretRef: postgresqlSetupJob-secret # secretKey: postgresqlSetupJob-password + # Add extra sidecar containers to job pod + extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always ## No code data migration datahubUpgrade: @@ -176,6 +196,17 @@ datahubUpgrade: securityContext: {} # runAsUser: 1000 podAnnotations: {} + # Add extra sidecar containers to job pod + extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always + cleanupJob: + # Add extra sidecar containers to job pod + extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always restoreIndices: resources: limits: @@ -184,6 +215,11 @@ datahubUpgrade: requests: cpu: 300m memory: 256Mi + # Add extra sidecar containers to job pod + extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always ## Runs system update processes ## Includes: Elasticsearch Indices Creation/Reindex (See global.elasticsearch.index for additional configuration) @@ -203,6 +239,11 @@ datahubSystemUpdate: requests: cpu: 300m memory: 256Mi + # Add extra sidecar containers to job pod + extraSidecars: [] + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always global: strict_mode: true From c1460c18053687ddfb1d817d00f28372fbb24605 Mon Sep 17 00:00:00 2001 From: KonstantinVishnivetskii <120646195+KonstantinVishnivetskii@users.noreply.github.com> Date: Fri, 19 May 2023 02:40:46 +0600 Subject: [PATCH 7/8] feat: values for PSQL db name, elastic prefix (#313) * Values for PSQL database name, elastic prefix --------- Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- charts/datahub/values.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 02b808b74..be6f7d7b5 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -179,6 +179,10 @@ postgresqlSetupJob: # - name: my-image-name # image: my-image # imagePullPolicy: Always + # Optionally set a specific database using extraEnvs + # extraEnvs: [] + # - name: "DATAHUB_DB_NAME" + # value: "dh" ## No code data migration datahubUpgrade: @@ -220,6 +224,10 @@ datahubUpgrade: # - name: my-image-name # image: my-image # imagePullPolicy: Always + # Optionally set a specific PostgreSQL database name using extraEnvs + # extraEnvs: + # - name: "DATAHUB_DB_NAME" + # value: "dh" ## Runs system update processes ## Includes: Elasticsearch Indices Creation/Reindex (See global.elasticsearch.index for additional configuration) @@ -257,6 +265,8 @@ global: skipcheck: "false" insecure: "false" useSSL: "false" + # If you want to specify index prefixes use indexPrefix + # indexPrefix: "dh" ## The following section controls when and how reindexing of elasticsearch indices are performed index: @@ -387,6 +397,12 @@ global: # secretKey: postgres-password # --------------OR---------------- # value: password + + # If you want to use specific PostgreSQL database use extraEnvs + # extraEnvs: + # - name: "DATAHUB_DB_NAME" + # value: "dh" + datahub: version: v0.10.2 From f5d8134b0e978d30f9d30437b771af29f9cd1e7c Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 26 May 2023 11:49:26 -0500 Subject: [PATCH 8/8] feat(search,schema-registry): updates for v0.10.3 release (#311) --- charts/datahub/Chart.yaml | 12 +-- charts/datahub/README.md | 4 +- .../acryl-datahub-actions/Chart.yaml | 2 +- .../templates/deployment.yaml | 5 + .../datahub/subcharts/datahub-gms/Chart.yaml | 2 +- .../datahub-gms/templates/configmap.yaml | 14 +++ .../datahub-gms/templates/deployment.yaml | 22 ++++- .../datahub/subcharts/datahub-gms/values.yaml | 2 +- .../subcharts/datahub-mae-consumer/Chart.yaml | 2 +- .../templates/deployment.yaml | 7 +- .../datahub-mae-consumer/values.yaml | 2 +- .../subcharts/datahub-mce-consumer/Chart.yaml | 2 +- .../templates/deployment.yaml | 7 +- .../datahub-mce-consumer/values.yaml | 2 +- .../templates/datahub-upgrade/_upgrade.tpl | 5 + .../datahub-system-update-job.yml | 8 ++ charts/datahub/templates/kafka-setup-job.yml | 7 ++ charts/datahub/values.yaml | 95 +++++++++++++++++-- charts/prerequisites/Chart.yaml | 2 +- charts/prerequisites/values.yaml | 3 +- 20 files changed, 175 insertions(+), 30 deletions(-) create mode 100644 charts/datahub/subcharts/datahub-gms/templates/configmap.yaml diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 7dc325c75..49866adc8 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,13 +4,13 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.164 +version: 0.2.165 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 0.10.2 +appVersion: 0.10.3 dependencies: - name: datahub-gms - version: 0.2.147 + version: 0.2.148 repository: file://./subcharts/datahub-gms condition: datahub-gms.enabled - name: datahub-frontend @@ -18,11 +18,11 @@ dependencies: repository: file://./subcharts/datahub-frontend condition: datahub-frontend.enabled - name: datahub-mae-consumer - version: 0.2.143 + version: 0.2.144 repository: file://./subcharts/datahub-mae-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-mce-consumer - version: 0.2.145 + version: 0.2.146 repository: file://./subcharts/datahub-mce-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-ingestion-cron @@ -30,7 +30,7 @@ dependencies: repository: file://./subcharts/datahub-ingestion-cron condition: datahub-ingestion-cron.enabled - name: acryl-datahub-actions - version: 0.2.136 + version: 0.2.137 repository: file://./subcharts/acryl-datahub-actions condition: acryl-datahub-actions.enabled maintainers: diff --git a/charts/datahub/README.md b/charts/datahub/README.md index 8a3e16bda..40a2dcc61 100644 --- a/charts/datahub/README.md +++ b/charts/datahub/README.md @@ -95,7 +95,7 @@ helm install datahub datahub/datahub --values <> | global.kafka.topics.metadata_change_log_versioned_topic_name | string | `"MetadataChangeLog_Versioned_v1"` | Kafka topic name for Versioned Metadata Change Log events | | global.kafka.topics.metadata_change_log_timeseries_topic_name | string | `"MetadataChangeLog_Timeseries_v1"` | Kafka topic name for Timeseries Metadata Change Log events | | global.kafka.topics.platform_event_topic_name | string | `"PlatformEvent_v1"` | Kafka topic name for Platform events | -| global.kafka.schemaregistry.url | string | `"http://prerequisites-cp-schema-registry:8081"` | URL to kafka schema registry | +| global.kafka.schemaregistry.url | string | `` | URL to kafka schema registry if using `KAFKA` type | | global.neo4j.host | string | `"prerequisites-neo4j:7474"` | Neo4j host address (with port) | | global.neo4j.uri | string | `"bolt://prerequisites-neo4j"` | Neo4j URI | | global.neo4j.username | string | `"neo4j"` | Neo4j user name | @@ -139,7 +139,7 @@ helm install datahub datahub/datahub --values <> | global.elasticsearch.auth.password.secretRef | string | `""` | Secret that contains the elasticsearch password | | global.elasticsearch.auth.password.secretKey | string | `""` | Secret key that contains the elasticsearch password | | global.elasticsearch.auth.password.value | string | `""` | Alternative to using the secret above, uses raw string value instead | -| global.kafka.schemaregistry.type | string | `"KAFKA"` | Type of schema registry (KAFKA or AWS_GLUE) | +| global.kafka.schemaregistry.type | string | `"INTERNAL"` | Type of schema registry (INTERNAL, KAFKA, or AWS_GLUE) | | global.kafka.schemaregistry.glue.region | string | `""` | Region of the AWS Glue schema registry | | global.kafka.schemaregistry.glue.registry | string | `""` | Name of the AWS Glue schema registry | | datahub.metadata_service_authentication.enabled | bool | `false` | Whether Metadata Service Authentication is enabled. | diff --git a/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml b/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml index 0c0e1a588..d2e46f93b 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.136 +version: 0.2.137 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.0.11 diff --git a/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml b/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml index aa1f79981..071916d55 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml @@ -73,8 +73,13 @@ spec: value: "{{ .Values.global.datahub.gms.port }}" - name: KAFKA_BOOTSTRAP_SERVER value: "{{ .Values.global.kafka.bootstrap.server }}" + {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} + - name: SCHEMA_REGISTRY_URL + value: {{ printf "http://%s-%s:%s/schema-registry/api/" .Release.Name "datahub-gms" .Values.global.datahub.gms.port }} + {{- else if eq .Values.global.kafka.schemaregistry.type "KAFKA" }} - name: SCHEMA_REGISTRY_URL value: "{{ .Values.global.kafka.schemaregistry.url }}" + {{- end }} - name: KAFKA_AUTO_OFFSET_POLICY value: "{{ .Values.actions.kafkaAutoOffsetPolicy }}" {{- if .Values.global.springKafkaConfigurationOverrides }} diff --git a/charts/datahub/subcharts/datahub-gms/Chart.yaml b/charts/datahub/subcharts/datahub-gms/Chart.yaml index ae513efc1..67cfa2aa4 100644 --- a/charts/datahub/subcharts/datahub-gms/Chart.yaml +++ b/charts/datahub/subcharts/datahub-gms/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for LinkedIn DataHub's datahub-gms component type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.147 +version: 0.2.148 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.10.0 diff --git a/charts/datahub/subcharts/datahub-gms/templates/configmap.yaml b/charts/datahub/subcharts/datahub-gms/templates/configmap.yaml new file mode 100644 index 000000000..1565e7a19 --- /dev/null +++ b/charts/datahub/subcharts/datahub-gms/templates/configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.global.elasticsearch.search.custom.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" .Release.Name "search-custom" }} + labels: + {{- include "datahub-gms.labels" . | nindent 4 }} + {{- range $key, $val := .Values.extraLabels }} + {{ $key }}: {{ $val | quote }} + {{- end }} +data: + search_config.yml: | + {{- toYaml .Values.global.elasticsearch.search.custom.config | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml index 4a9be56b7..190dcf20e 100644 --- a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml @@ -47,6 +47,11 @@ spec: defaultMode: 0444 secretName: {{ .name }} {{- end }} + {{- if .Values.global.elasticsearch.search.custom.enabled }} + - configMap: + name: {{ printf "%s-%s" .Release.Name "search-custom" }} + name: search-config + {{- end }} {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} @@ -89,6 +94,12 @@ spec: periodSeconds: {{ .Values.readinessProbe.periodSeconds }} failureThreshold: {{ .Values.readinessProbe.failureThreshold }} env: + {{- if .Values.global.elasticsearch.search.custom.enabled }} + - name: ELASTICSEARCH_QUERY_CUSTOM_CONFIG_ENABLED + value: "true" + - name: ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE + value: "/datahub/datahub-gms/resources/search/search_config.yml" + {{- end}} {{- if gt .Values.replicaCount 1.0}} - name: SEARCH_SERVICE_CACHE_IMPLEMENTATION value: "hazelcast" @@ -137,9 +148,12 @@ spec: value: "{{ .Values.global.sql.datasource.driver }}" - name: KAFKA_BOOTSTRAP_SERVER value: "{{ .Values.global.kafka.bootstrap.server }}" - {{- with .Values.global.kafka.schemaregistry.url }} + {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} - name: KAFKA_SCHEMAREGISTRY_URL - value: "{{ . }}" + value: {{ printf "http://localhost:%s/schema-registry/api/" .Values.global.datahub.gms.port }} + {{- else if eq .Values.global.kafka.schemaregistry.type "KAFKA" }} + - name: KAFKA_SCHEMAREGISTRY_URL + value: "{{ .Values.global.kafka.schemaregistry.url }}" {{- end }} {{- with .Values.global.kafka.schemaregistry.type }} - name: SCHEMA_REGISTRY_TYPE @@ -331,6 +345,10 @@ spec: - name: datahub-certs-dir mountPath: {{ .path | default "/mnt/certs" }} {{- end }} + {{- if .Values.global.elasticsearch.search.custom.enabled }} + - name: search-config + mountPath: "/datahub/datahub-gms/resources/search" + {{- end }} {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/datahub/subcharts/datahub-gms/values.yaml b/charts/datahub/subcharts/datahub-gms/values.yaml index fc06079fd..1056df635 100644 --- a/charts/datahub/subcharts/datahub-gms/values.yaml +++ b/charts/datahub/subcharts/datahub-gms/values.yaml @@ -200,7 +200,7 @@ global: enable_retention: false ## Set to true to enable retention on local DB ## Enables always emitting a MCL even when no changes are detected. Used for Time Based Lineage when no changes occur. - alwaysEmitChangeLog: true + alwaysEmitChangeLog: false ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading enableGraphDiffMode: true diff --git a/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml index fd7d46e1f..0fdbf6b62 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml +++ b/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.143 +version: 0.2.144 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.10.0 diff --git a/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml index 4b6e1ad3d..c069074b8 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml @@ -102,9 +102,12 @@ spec: value: "{{ .Values.global.datahub.gms.port }}" - name: KAFKA_BOOTSTRAP_SERVER value: "{{ .Values.global.kafka.bootstrap.server }}" - {{- with .Values.global.kafka.schemaregistry.url }} + {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} - name: KAFKA_SCHEMAREGISTRY_URL - value: "{{ . }}" + value: {{ printf "http://%s-%s:%s/schema-registry/api/" .Release.Name "datahub-gms" .Values.global.datahub.gms.port }} + {{- else if eq .Values.global.kafka.schemaregistry.type "KAFKA" }} + - name: KAFKA_SCHEMAREGISTRY_URL + value: "{{ .Values.global.kafka.schemaregistry.url }}" {{- end }} {{- with .Values.global.kafka.schemaregistry.type }} - name: SCHEMA_REGISTRY_TYPE diff --git a/charts/datahub/subcharts/datahub-mae-consumer/values.yaml b/charts/datahub/subcharts/datahub-mae-consumer/values.yaml index 2a2b91ea9..f9104089c 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/values.yaml +++ b/charts/datahub/subcharts/datahub-mae-consumer/values.yaml @@ -207,7 +207,7 @@ global: # secretKey: ## Enables always emitting a MCL even when no changes are detected. Used for Time Based Lineage when no changes occur. - alwaysEmitChangeLog: true + alwaysEmitChangeLog: false ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading enableGraphDiffMode: true diff --git a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml index 241d0827d..85eeabf88 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml +++ b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.145 +version: 0.2.146 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.10.0 diff --git a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml index 1fea2719a..dff8a2d58 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml @@ -98,9 +98,12 @@ spec: value: "true" - name: KAFKA_BOOTSTRAP_SERVER value: "{{ .Values.global.kafka.bootstrap.server }}" - {{- with .Values.global.kafka.schemaregistry.url }} + {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} - name: KAFKA_SCHEMAREGISTRY_URL - value: "{{ . }}" + value: {{ printf "http://%s-%s:%s/schema-registry/api/" .Release.Name "datahub-gms" .Values.global.datahub.gms.port }} + {{- else if eq .Values.global.kafka.schemaregistry.type "KAFKA" }} + - name: KAFKA_SCHEMAREGISTRY_URL + value: "{{ .Values.global.kafka.schemaregistry.url }}" {{- end }} {{- with .Values.global.kafka.schemaregistry.type }} - name: SCHEMA_REGISTRY_TYPE diff --git a/charts/datahub/subcharts/datahub-mce-consumer/values.yaml b/charts/datahub/subcharts/datahub-mce-consumer/values.yaml index 75769726a..892b391d2 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/values.yaml +++ b/charts/datahub/subcharts/datahub-mce-consumer/values.yaml @@ -200,7 +200,7 @@ global: secretKey: "mysql-password" ## Enables always emitting a MCL even when no changes are detected. Used for Time Based Lineage when no changes occur. - alwaysEmitChangeLog: true + alwaysEmitChangeLog: false ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading enableGraphDiffMode: true diff --git a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl index 8d784f50e..eec1df561 100644 --- a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl +++ b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl @@ -42,8 +42,13 @@ Return the env variables for upgrade jobs {{- end }} - name: KAFKA_BOOTSTRAP_SERVER value: "{{ .Values.global.kafka.bootstrap.server }}" +{{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} +- name: KAFKA_SCHEMAREGISTRY_URL + value: {{ printf "http://%s-%s:%s/schema-registry/api/" .Release.Name "datahub-gms" .Values.global.datahub.gms.port }} +{{- else if eq .Values.global.kafka.schemaregistry.type "KAFKA" }} - name: KAFKA_SCHEMAREGISTRY_URL value: "{{ .Values.global.kafka.schemaregistry.url }}" +{{- end }} - name: ELASTICSEARCH_HOST value: {{ .Values.global.elasticsearch.host | quote }} - name: ELASTICSEARCH_PORT diff --git a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml index 29786df8e..54a946ad2 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml @@ -71,6 +71,14 @@ spec: {{- include "datahub.upgrade.env" . | nindent 12}} - name: DATAHUB_ANALYTICS_ENABLED value: {{ .Values.global.datahub_analytics_enabled | quote }} + {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} + - name: SCHEMA_REGISTRY_SYSTEM_UPDATE + value: "true" + - name: SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS + value: "false" + - name: SPRING_KAFKA_PROPERTIES_USE_LATEST_VERSION + value: "true" + {{- end }} {{- with .Values.global.kafka.schemaregistry.type }} - name: SCHEMA_REGISTRY_TYPE value: "{{ . }}" diff --git a/charts/datahub/templates/kafka-setup-job.yml b/charts/datahub/templates/kafka-setup-job.yml index 86bc43980..100dd9dec 100644 --- a/charts/datahub/templates/kafka-setup-job.yml +++ b/charts/datahub/templates/kafka-setup-job.yml @@ -63,6 +63,13 @@ spec: value: {{ .Values.global.kafka.zookeeper.server | quote }} - name: KAFKA_BOOTSTRAP_SERVER value: {{ .Values.global.kafka.bootstrap.server | quote }} + {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} + - name: USE_CONFLUENT_SCHEMA_REGISTRY + value: "false" + {{- else if eq .Values.global.kafka.schemaregistry.type "KAFKA" }} + - name: USE_CONFLUENT_SCHEMA_REGISTRY + value: "true" + {{- end }} {{- if .Values.global.springKafkaConfigurationOverrides }} {{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }} - name: KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }} diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index be6f7d7b5..dddea2988 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -333,6 +333,81 @@ global: ## graph dao max result size maxResult: 10000 + custom: + enabled: false + # See documentation: https://datahubproject.io/docs/how/search/#customizing-search + config: + # Notes: + # + # First match wins + # + # queryRegex = Java regex syntax + # + # functionScores - See the following for function score syntax + # https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-function-score-query.html + + queryConfigurations: + # Select * + - queryRegex: '[*]|' + simpleQuery: false + prefixMatchQuery: false + exactMatchQuery: false + boolQuery: + must_not: + term: + deprecated: + value: true + functionScore: + functions: + - filter: + term: + materialized: + value: true + weight: 0.8 + score_mode: multiply + boost_mode: multiply + + # Criteria for exact-match only + # Contains quoted or contains underscore then use exact match query + - queryRegex: >- + ["'].+["']|\S+_\S+ + simpleQuery: false + prefixMatchQuery: true + exactMatchQuery: true + functionScore: + functions: + - filter: + term: + materialized: + value: true + weight: 0.8 + - filter: + term: + deprecated: + value: true + weight: 0 + score_mode: multiply + boost_mode: multiply + # default + - queryRegex: .* + simpleQuery: true + prefixMatchQuery: true + exactMatchQuery: true + boolQuery: + must_not: + term: + deprecated: + value: true + functionScore: + functions: + - filter: + term: + materialized: + value: true + weight: 0.8 + score_mode: multiply + boost_mode: multiply + kafka: bootstrap: server: "prerequisites-kafka:9092" @@ -355,8 +430,14 @@ global: # partitions: 3 # replicationFactor: 3 schemaregistry: - url: "http://prerequisites-cp-schema-registry:8081" - type: KAFKA + # GMS Implementation - `url` configured based on component context + type: INTERNAL + # Confluent Kafka Implementation + # type: KAFKA + # url: "http://prerequisites-cp-schema-registry:8081" + + # Glue Implementation - `url` not applicable + # type: AWS_GLUE # glue: # region: us-east-1 # registry: datahub @@ -397,15 +478,15 @@ global: # secretKey: postgres-password # --------------OR---------------- # value: password - + # If you want to use specific PostgreSQL database use extraEnvs # extraEnvs: # - name: "DATAHUB_DB_NAME" - # value: "dh" + # value: "dh" datahub: - version: v0.10.2 + version: v0.10.3 gms: port: "8080" nodePort: "30001" @@ -436,7 +517,7 @@ global: managed_ingestion: enabled: true - defaultCliVersion: "0.10.0" + defaultCliVersion: "0.10.3" metadata_service_authentication: enabled: false @@ -462,7 +543,7 @@ global: # salt: ## Enables always emitting a MCL even when no changes are detected. Used for Time Based Lineage when no changes occur. - alwaysEmitChangeLog: true + alwaysEmitChangeLog: false ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading enableGraphDiffMode: true diff --git a/charts/prerequisites/Chart.yaml b/charts/prerequisites/Chart.yaml index 1fbf1443e..69c5871b9 100644 --- a/charts/prerequisites/Chart.yaml +++ b/charts/prerequisites/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for packages that Datahub depends on type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.0.15 +version: 0.0.16 dependencies: - name: elasticsearch version: 7.17.3 diff --git a/charts/prerequisites/values.yaml b/charts/prerequisites/values.yaml index 1968f5481..43d57c373 100644 --- a/charts/prerequisites/values.yaml +++ b/charts/prerequisites/values.yaml @@ -66,9 +66,10 @@ postgresql: existingSecret: postgresql-secrets cp-helm-charts: + enabled: false # Schema registry is under the community license cp-schema-registry: - enabled: true + enabled: false kafka: bootstrapServers: "prerequisites-kafka:9092" # <>-kafka:9092 cp-kafka: