From dd2678bdfcb98b156a947eb0a40df93d20154586 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Wed, 25 Jan 2023 15:12:05 +0000 Subject: [PATCH] fix(kafka-setup): Make kafka setup use explicit topic names (#245) * fix(kafka-setup): Make kafka setup use explicit topic names * Set kafka env vars for all deployments * Update gms & actions pod env vars * Add documentation for new properties --- charts/datahub/Chart.yaml | 10 +++++----- charts/datahub/README.md | 9 +++++++++ .../acryl-datahub-actions/Chart.yaml | 2 +- .../templates/deployment.yaml | 20 +++++++++++++++++++ .../datahub/subcharts/datahub-gms/Chart.yaml | 2 +- .../datahub-gms/templates/deployment.yaml | 20 +++++++++++++++++++ .../subcharts/datahub-mae-consumer/Chart.yaml | 2 +- .../templates/deployment.yaml | 12 +++++++++++ .../subcharts/datahub-mce-consumer/Chart.yaml | 2 +- .../templates/deployment.yaml | 14 +++++++++++++ .../templates/datahub-upgrade/_upgrade.tpl | 16 +++++++++++++++ charts/datahub/templates/kafka-setup-job.yml | 20 +++++++++++++++++++ charts/datahub/values.yaml | 12 +++++++++++ 13 files changed, 132 insertions(+), 9 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 3dd35b1a2..089554f96 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,13 +4,13 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.140 +version: 0.2.141 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.9.6.1 dependencies: - name: datahub-gms - version: 0.2.132 + version: 0.2.133 repository: file://./subcharts/datahub-gms condition: datahub-gms.enabled - name: datahub-frontend @@ -18,11 +18,11 @@ dependencies: repository: file://./subcharts/datahub-frontend condition: datahub-frontend.enabled - name: datahub-mae-consumer - version: 0.2.131 + version: 0.2.132 repository: file://./subcharts/datahub-mae-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-mce-consumer - version: 0.2.132 + version: 0.2.133 repository: file://./subcharts/datahub-mce-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-ingestion-cron @@ -30,7 +30,7 @@ dependencies: repository: file://./subcharts/datahub-ingestion-cron condition: datahub-ingestion-cron.enabled - name: acryl-datahub-actions - version: 0.2.130 + version: 0.2.131 repository: file://./subcharts/acryl-datahub-actions condition: acryl-datahub-actions.enabled maintainers: diff --git a/charts/datahub/README.md b/charts/datahub/README.md index a749aec77..67801a702 100644 --- a/charts/datahub/README.md +++ b/charts/datahub/README.md @@ -79,6 +79,15 @@ helm install datahub datahub/datahub --values <> | global.elasticsearch.port | string | `"9200"` | Elasticsearch port | | global.kafka.bootstrap.server | string | `"prerequisites-broker:9092"` | Kafka bootstrap servers (with port) | | global.kafka.zookeeper.server | string | `"prerequisites-zookeeper:2181"` | Kafka zookeeper servers (with port) | +| global.kafka.topics.metadata_change_event_name | string | `"MetadataChangeEvent_v4"` | Kafka topic name for Metadata Change Events (deprecated) | +| global.kafka.topics.failed_metadata_change_event_name | string | `"FailedMetadataChangeEvent_v4"` | Kafka topic name for Failed Metadata Change events (deprecated) | +| global.kafka.topics.metadata_audit_event_name | string | `"MetadataAuditEvent_v4"` | Kafka topic name for Metadata Audit events (deprecated) | +| global.kafka.topics.datahub_usage_event_name | string | `"DataHubUsageEvent_v1"` | Kafka topic name for DataHub Usage events | +| global.kafka.topics.metadata_change_proposal_topic_name | string | `"MetadataChangeProposal_v1"` | Kafka topic name for Metadata Change Proposal events | +| global.kafka.topics.failed_metadata_change_proposal_topic_name | string | `"FailedMetadataChangeProposal_v1"` | Kafka topic name for Failed Metadata Change Proposal events | +| global.kafka.topics.metadata_change_log_versioned_topic_name | string | `"MetadataChangeLog_Versioned_v1"` | Kafka topic name for Versioned Metadata Change Log events | +| global.kafka.topics.metadata_change_log_timeseries_topic_name | string | `"MetadataChangeLog_Timeseries_v1"` | Kafka topic name for Timeseries Metadata Change Log events | +| global.kafka.topics.platform_event_topic_name | string | `"PlatformEvent_v1"` | Kafka topic name for Platform events | | global.kafka.schemaregistry.url | string | `"http://prerequisites-cp-schema-registry:8081"` | URL to kafka schema registry | | global.neo4j.host | string | `"prerequisites-neo4j:7474"` | Neo4j host address (with port) | | global.neo4j.uri | string | `"bolt://prerequisites-neo4j"` | Neo4j URI | diff --git a/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml b/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml index 7faac55b9..a30ee8174 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.130 +version: 0.2.131 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.0.7 diff --git a/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml b/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml index fee5f1855..5e652ccb5 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml @@ -89,6 +89,26 @@ spec: key: {{ $envVarValue }} {{- end }} {{- end }} + {{- with .Values.global.kafka.topics }} + - name: METADATA_CHANGE_EVENT_NAME + value: {{ .metadata_change_event_name }} + - name: FAILED_METADATA_CHANGE_EVENT_NAME + value: {{ .failed_metadata_change_event_name }} + - name: METADATA_AUDIT_EVENT_NAME + value: {{ .metadata_audit_event_name }} + - name: DATAHUB_USAGE_EVENT_NAME + value: {{ .datahub_usage_event_name }} + - name: METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: {{ .metadata_change_proposal_topic_name }} + - name: FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: {{ .failed_metadata_change_proposal_topic_name }} + - name: METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME + value: {{ .metadata_change_log_versioned_topic_name }} + - name: METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME + value: {{ .metadata_change_log_timeseries_topic_name }} + - name: PLATFORM_EVENT_TOPIC_NAME + value: {{ .platform_event_topic_name }} + {{- end }} {{- if .Values.global.datahub.metadata_service_authentication.enabled }} - name: DATAHUB_SYSTEM_CLIENT_ID value: {{ .Values.global.datahub.metadata_service_authentication.systemClientId }} diff --git a/charts/datahub/subcharts/datahub-gms/Chart.yaml b/charts/datahub/subcharts/datahub-gms/Chart.yaml index ade272ad8..8b5c8fba9 100644 --- a/charts/datahub/subcharts/datahub-gms/Chart.yaml +++ b/charts/datahub/subcharts/datahub-gms/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for LinkedIn DataHub's datahub-gms component type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.132 +version: 0.2.133 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.9.6.1 diff --git a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml index d952e5c55..e7fbfe96a 100644 --- a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml @@ -186,6 +186,26 @@ spec: key: {{ $envVarValue }} {{- end }} {{- end }} + {{- with .Values.global.kafka.topics }} + - name: METADATA_CHANGE_EVENT_NAME + value: {{ .metadata_change_event_name }} + - name: FAILED_METADATA_CHANGE_EVENT_NAME + value: {{ .failed_metadata_change_event_name }} + - name: METADATA_AUDIT_EVENT_NAME + value: {{ .metadata_audit_event_name }} + - name: DATAHUB_USAGE_EVENT_NAME + value: {{ .datahub_usage_event_name }} + - name: METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: {{ .metadata_change_proposal_topic_name }} + - name: FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: {{ .failed_metadata_change_proposal_topic_name }} + - name: METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME + value: {{ .metadata_change_log_versioned_topic_name }} + - name: METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME + value: {{ .metadata_change_log_timeseries_topic_name }} + - name: PLATFORM_EVENT_TOPIC_NAME + value: {{ .platform_event_topic_name }} + {{- end }} {{- if .Values.global.datahub.metadata_service_authentication.enabled }} - name: METADATA_SERVICE_AUTH_ENABLED value: "true" diff --git a/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml index 8031967be..bd561fdcf 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml +++ b/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.131 +version: 0.2.132 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.9.6.1 diff --git a/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml index aeefe92be..985d563c5 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml @@ -190,6 +190,18 @@ spec: key: {{ $envVarValue }} {{- end }} {{- end }} + {{- with .Values.global.kafka.topics }} + - name: METADATA_AUDIT_EVENT_NAME + value: {{ .metadata_audit_event_name }} + - name: DATAHUB_USAGE_EVENT_NAME + value: {{ .datahub_usage_event_name }} + - name: METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME + value: {{ .metadata_change_log_versioned_topic_name }} + - name: METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME + value: {{ .metadata_change_log_timeseries_topic_name }} + - name: PLATFORM_EVENT_TOPIC_NAME + value: {{ .platform_event_topic_name }} + {{- end }} {{- with .Values.extraEnvs }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml index 42934a752..8b500a444 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml +++ b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.132 +version: 0.2.133 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.9.6.1 diff --git a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml index 95c68f147..68a533e37 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml @@ -185,6 +185,20 @@ spec: key: {{ $envVarValue }} {{- end }} {{- end }} + {{- with .Values.global.kafka.topics }} + - name: METADATA_CHANGE_EVENT_NAME + value: {{ .metadata_change_event_name }} + - name: FAILED_METADATA_CHANGE_EVENT_NAME + value: {{ .failed_metadata_change_event_name }} + - name: METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: {{ .metadata_change_proposal_topic_name }} + - name: FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: {{ .failed_metadata_change_proposal_topic_name }} + - name: METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME + value: {{ .metadata_change_log_versioned_topic_name }} + - name: METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME + value: {{ .metadata_change_log_timeseries_topic_name }} + {{- end }} {{- with .Values.extraEnvs }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl index b17a23485..8aefc98f9 100644 --- a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl +++ b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl @@ -98,4 +98,20 @@ Return the env variables for upgrade jobs key: {{ $envVarValue }} {{- end }} {{- end }} +{{- with .Values.global.kafka.topics }} +- name: METADATA_CHANGE_EVENT_NAME + value: {{ .metadata_change_event_name }} +- name: FAILED_METADATA_CHANGE_EVENT_NAME + value: {{ .failed_metadata_change_event_name }} +- name: METADATA_AUDIT_EVENT_NAME + value: {{ .metadata_audit_event_name }} +- name: METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: {{ .metadata_change_proposal_topic_name }} +- name: FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: {{ .failed_metadata_change_proposal_topic_name }} +- name: METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME + value: {{ .metadata_change_log_versioned_topic_name }} +- name: METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME + value: {{ .metadata_change_log_timeseries_topic_name }} +{{- end }} {{- end -}} \ No newline at end of file diff --git a/charts/datahub/templates/kafka-setup-job.yml b/charts/datahub/templates/kafka-setup-job.yml index dd9bbea83..8f16a44ff 100644 --- a/charts/datahub/templates/kafka-setup-job.yml +++ b/charts/datahub/templates/kafka-setup-job.yml @@ -78,6 +78,26 @@ spec: key: {{ $envVarValue }} {{- end }} {{- end }} + {{- with .Values.global.kafka.topics }} + - name: METADATA_CHANGE_EVENT_NAME + value: {{ .metadata_change_event_name }} + - name: FAILED_METADATA_CHANGE_EVENT_NAME + value: {{ .failed_metadata_change_event_name }} + - name: METADATA_AUDIT_EVENT_NAME + value: {{ .metadata_audit_event_name }} + - name: DATAHUB_USAGE_EVENT_NAME + value: {{ .datahub_usage_event_name }} + - name: METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: {{ .metadata_change_proposal_topic_name }} + - name: FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: {{ .failed_metadata_change_proposal_topic_name }} + - name: METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME + value: {{ .metadata_change_log_versioned_topic_name }} + - name: METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME + value: {{ .metadata_change_log_timeseries_topic_name }} + - name: PLATFORM_EVENT_TOPIC_NAME + value: {{ .platform_event_topic_name }} + {{- end }} {{- with .Values.global.kafka.partitions }} - name: PARTITIONS value: {{ . | quote }} diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 0876ae647..2585c274d 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -256,6 +256,18 @@ global: server: "prerequisites-kafka:9092" zookeeper: server: "prerequisites-zookeeper:2181" + # This section defines the names for the kafka topics that DataHub depends on, at a global level. Do not override this config + # at a sub-chart level. + topics: + metadata_change_event_name: "MetadataChangeEvent_v4" + failed_metadata_change_event_name: "FailedMetadataChangeEvent_v4" + metadata_audit_event_name: "MetadataAuditEvent_v4" + datahub_usage_event_name: "DataHubUsageEvent_v1" + metadata_change_proposal_topic_name: "MetadataChangeProposal_v1" + failed_metadata_change_proposal_topic_name: "FailedMetadataChangeProposal_v1" + metadata_change_log_versioned_topic_name: "MetadataChangeLog_Versioned_v1" + metadata_change_log_timeseries_topic_name: "MetadataChangeLog_Timeseries_v1" + platform_event_topic_name: "PlatformEvent_v1" ## For AWS MSK set this to a number larger than 1 # partitions: 3 # replicationFactor: 3