Add compose file for multi-cluster setup

cadence-workflow · Mar 4, 2021 · 05d35be · 05d35be
1 parent a278995
commit 05d35be
Show file tree

Hide file tree

Showing 3 changed files with 93 additions and 8 deletions.
diff --git a/docker/README.md b/docker/README.md
@@ -31,6 +31,7 @@ We also provide several other compose files for different features/modes:
 * docker-compose-mysql.yml uses MySQL as persistence storage
 * docker-compose-postgres.yml uses PosstgreSQL as persistence storage
 * docker-compose-prometheus.yaml runs with Prometheus
+* docker-compose-multiclusters.yaml runs with 2 cadence clusters
 
 Also feel free to make your own to combine the above features.
 

diff --git a/docker/config_template.yaml b/docker/config_template.yaml
@@ -117,7 +117,7 @@ services:
         metrics:
             statsd:
                 hostPort: {{ .Env.STATSD_ENDPOINT }}
-                prefix: "cadence-frontend"
+                prefix: {{ default .Env.STATSD_FRONTEND_PREFIX "cadence-frontend" }}
         {{- else if .Env.PROMETHEUS_ENDPOINT }}
         metrics:
             prometheus:
@@ -138,7 +138,7 @@ services:
         metrics:
             statsd:
                 hostPort: {{ .Env.STATSD_ENDPOINT }}
-                prefix: "cadence-matching"
+                prefix: {{ default .Env.STATSD_MATCHING_PREFIX "cadence-matching" }}
         {{- else if .Env.PROMETHEUS_ENDPOINT }}
         metrics:
             prometheus:
@@ -159,7 +159,7 @@ services:
         metrics:
             statsd:
                 hostPort: {{ .Env.STATSD_ENDPOINT }}
-                prefix: "cadence-history"
+                prefix: {{ default .Env.STATSD_HISTORY_PREFIX "cadence-history" }}
         {{- else if .Env.PROMETHEUS_ENDPOINT }}
         metrics:
             prometheus:
@@ -180,7 +180,7 @@ services:
         metrics:
             statsd:
                 hostPort: {{ .Env.STATSD_ENDPOINT }}
-                prefix: "cadence-worker"
+                prefix: {{ default .Env.STATSD_WORKER_PREFIX "cadence-worker" }}
         {{- else if .Env.PROMETHEUS_ENDPOINT }}
         metrics:
             prometheus:
@@ -196,14 +196,26 @@ services:
 clusterMetadata:
     enableGlobalDomain: {{ default .Env.ENABLE_GLOBAL_DOMAIN "false" }}
     failoverVersionIncrement: 10
-    masterClusterName: "master"
-    currentClusterName: "master"
+    masterClusterName: "primary"
+    {{- if .Env.IS_NOT_PRIMARY }}
+    currentClusterName: "secondary"
+    {{- else }}
+    currentClusterName: "primary"
+    {{- end }}
     clusterInformation:
-        master:
+        primary:
             enabled: true
             initialFailoverVersion: 0
             rpcName: "cadence-frontend"
-            rpcAddress: {{ default .Env.BIND_ON_IP "127.0.0.1" }}:7933
+            rpcAddress: {{ default .Env.PRIMARY_SEEDS "cadence" }}:7933
+        {{- if .Env.ENABLE_GLOBAL_DOMAIN }}
+        secondary:
+            enabled: true
+            initialFailoverVersion: 2
+            rpcName: "cadence-frontend"
+            rpcAddress: {{ default .Env.SECONDARY_SEEDS "cadence-secondary" }}:7933
+        {{- end }}
+
 
 dcRedirectionPolicy:
     policy: {{ default .Env.DC_REDIRECT_POLICY "selected-apis-forwarding" }}

diff --git a/docker/docker-compose-multiclusters.yml b/docker/docker-compose-multiclusters.yml
@@ -0,0 +1,72 @@
+version: '3'
+services:
+  cassandra:
+    image: cassandra:3.11
+    ports:
+      - "9042:9042"
+  statsd:
+    image: graphiteapp/graphite-statsd
+    ports:
+      - "8080:80"
+      - "2003:2003"
+      - "8125:8125"
+      - "8126:8126"
+  cadence:
+    image: ubercadence/server:master-auto-setup
+    ports:
+     - "7933:7933"
+     - "7934:7934"
+     - "7935:7935"
+     - "7939:7939"
+    environment:
+      - "CASSANDRA_SEEDS=cassandra"
+      - "STATSD_ENDPOINT=statsd:8125"
+      - "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml"
+      - "ENABLE_GLOBAL_DOMAIN=true"
+      - "KEYSPACE=cadence_primary"
+      - "VISIBILITY_KEYSPACE=cadence_visibility_primary"
+      - "STATSD_FRONTEND_PREFIX=cadence-frontend-primary"
+      - "STATSD_MATCHING_PREFIX=cadence-matching-primary"
+      - "STATSD_HISTORY_PREFIX=cadence-history-primary"
+      - "STATSD_WORKER_PREFIX=cadence-worker-primary"
+    depends_on:
+      - cassandra
+      - statsd
+  cadence-secondary:
+    image: ubercadence/server:master-auto-setup
+    ports:
+     - "7943:7933"
+     - "7944:7934"
+     - "7945:7935"
+     - "7949:7939"
+    environment:
+      - "CASSANDRA_SEEDS=cassandra"
+      - "STATSD_ENDPOINT=statsd:8125"
+      - "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml"
+      - "IS_NOT_PRIMARY=true"
+      - "ENABLE_GLOBAL_DOMAIN=true"
+      - "KEYSPACE=cadence_secondary"
+      - "VISIBILITY_KEYSPACE=cadence_visibility_secondary"
+      - "STATSD_FRONTEND_PREFIX=cadence-frontend-secondary"
+      - "STATSD_MATCHING_PREFIX=cadence-matching-secondary"
+      - "STATSD_HISTORY_PREFIX=cadence-history-secondary"
+      - "STATSD_WORKER_PREFIX=cadence-worker-secondary"
+    depends_on:
+      - cassandra
+      - statsd
+  cadence-web:
+    image: ubercadence/web:latest
+    environment:
+      - "CADENCE_TCHANNEL_PEERS=cadence:7933"
+    ports:
+      - "8088:8088"
+    depends_on:
+      - cadence
+  cadence-web-secondary:
+    image: ubercadence/web:latest
+    environment:
+      - "CADENCE_TCHANNEL_PEERS=cadence-secondary:7933"
+    ports:
+      - "8089:8088"
+    depends_on:
+      - cadence-secondary