From 3151f16805f81191e56951c10d7bb4df55f8b8d3 Mon Sep 17 00:00:00 2001 From: "hunter.elbourn@cloudkitchens.com" Date: Wed, 17 Aug 2022 00:50:12 -0700 Subject: [PATCH 1/2] (datahub-upgrade): Restore Indices Resources This PR adds the ability for end-users to specify resource requests and limits for the datahub upgrade job 'restore indices'. This is needed for situations where the restore indices job will OOM due to a large backfill of data. I've seen this happening with our deployment of datahub and am unable to tune the resources for the restore indices job because the current requests and limits are hard-coded. In addition, bumps chart version to 0.2.90 --- charts/datahub/Chart.yaml | 4 ++-- charts/datahub/README.md | 1 + .../datahub-restore-indices-job-template.yml | 7 +------ charts/datahub/values.yaml | 8 ++++++++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index eb260475e..1d90eaee3 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,10 +4,10 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.89 +version: 0.2.90 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 0.8.43 +appVersion: 0.8.44 dependencies: - name: datahub-gms version: 0.2.5 diff --git a/charts/datahub/README.md b/charts/datahub/README.md index 7817e6f8f..fbccf4ea7 100644 --- a/charts/datahub/README.md +++ b/charts/datahub/README.md @@ -34,6 +34,7 @@ helm install datahub datahub/datahub --values <> | datahubUpgrade.podSecurityContext | object | `{}` | Pod security context for datahubUpgrade jobs | | datahubUpgrade.securityContext | object | `{}` | Container security context for datahubUpgrade jobs | | datahubUpgrade.podAnnotations | object | `{}` | Pod annotations for datahubUpgrade jobs | +| datahubUpgrade.restoreIndices.resources | object | '{}' | Kube Resource definitions for the datahub upgrade job 'restore indices' | | elasticsearchSetupJob.enabled | bool | `true` | Enable setup job for elasicsearch | | elasticsearchSetupJob.image.repository | string | `"linkedin/datahub-elasticsearch-setup"` | Image repository for elasticsearchSetupJob | | elasticsearchSetupJob.image.tag | string | `"v0.8.43"` | Image repository for elasticsearchSetupJob | diff --git a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml index d47dcab7a..884edf3eb 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml @@ -83,12 +83,7 @@ spec: {{- toYaml . | nindent 16 }} {{- end }} resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 300m - memory: 256Mi + {{- toYaml .Values.datahubUpgrade.restoreIndices.resources | nindent 16}} {{- with .Values.datahubUpgrade.nodeSelector }} nodeSelector: {{- toYaml . | nindent 12 }} diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 0e38c0771..81d486270 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -99,6 +99,14 @@ datahubUpgrade: securityContext: {} # runAsUser: 1000 podAnnotations: {} + restoreIndices: + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 300m + memory: 256Mi global: graph_service_impl: neo4j From b53bc68971173b6e9f0c0b1b810de5193d9d3cfc Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 23 Aug 2022 19:52:22 +0530 Subject: [PATCH 2/2] Update Chart.yaml --- charts/datahub/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 1d90eaee3..3bb50482b 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.90 +version: 0.2.91 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.8.44