Feat/elasticsearch optimization ext (#229)

* feat(elasticsearch): allow document count mismatch override
acryldata · Jan 3, 2023 · e5a9342 · e5a9342
1 parent 2b637bc
commit e5a9342
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 2 deletions.
diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml
@@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.127
+version: 0.2.128
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.9.5

diff --git a/charts/datahub/templates/datahub-upgrade/datahub-build-indices-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-build-indices-job.yml
@@ -152,6 +152,10 @@ spec:
             - name: ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS
               value: {{ . | quote }}
             {{- end }}
+            {{- with .Values.global.elasticsearch.index.upgrade.allowDocCountMismatch }}
+            - name: ELASTICSEARCH_BUILD_INDICES_ALLOW_DOC_COUNT_MISMATCH
+              value: {{ . | quote }}
+            {{- end }}
             - name: GRAPH_SERVICE_IMPL
               value: {{ .Values.global.graph_service_impl }}
             {{- if eq .Values.global.graph_service_impl "neo4j" }}

diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml
@@ -215,9 +215,18 @@ global:
         enabled: true
 
         ## When reindexing is required, this option will clone the existing index as a backup
-        ## The clone indices are not currently managed
+        ## The clone indices are not currently managed.
         # cloneIndices: true
 
+        ## Typically when reindexing the document counts between the original and destination indices should match.
+        ## In some cases reindexing might not be able to proceed due to incompatibilities between a document in the
+        ## orignal index and the new index's mappings. This document could be dropped and re-ingested or restored from
+        ## the SQL database.
+        ##
+        ## This setting allows continuing if and only if the cloneIndices setting is also enabled which
+        ## ensures a complete backup of the original index is preserved.
+        # allowDocCountMismatch: false
+
   kafka:
     bootstrap:
       server: "prerequisites-kafka:9092"