Docker compose setup for async workflows with kafka queue

Dockerfile

@@ -85,7 +85,7 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD /start-cadence.sh
 
 
-# All-in-one Cadence server
+# All-in-one Cadence server (~450mb)
 FROM cadence-server AS cadence-auto-setup
 
 RUN apk add --update --no-cache ca-certificates py3-pip mysql-client
@@ -95,6 +95,14 @@ COPY docker/start.sh /start.sh
 
 CMD /start.sh
 
+# All-in-one Cadence server with Kafka (~550mb)
+FROM cadence-auto-setup AS cadence-auto-setup-with-kafka
+
+RUN apk add openjdk11
+RUN wget https://archive.apache.org/dist/kafka/2.1.1/kafka_2.12-2.1.1.tgz -O kafka.tgz
+RUN mkdir -p kafka
+RUN tar -xvzf kafka.tgz --strip 1 -C kafka
+ENV KAFKA_HOME /etc/cadence/kafka
 
 # Cadence CLI
 FROM alpine AS cadence-cli

config/development_async_wf_kafka_queue.yaml

@@ -0,0 +1,139 @@
+persistence:
+  defaultStore: cass-default
+  visibilityStore: cass-visibility
+  numHistoryShards: 4
+  datastores:
+    cass-default:
+      nosql:
+        pluginName: "cassandra"
+        hosts: "127.0.0.1"
+        keyspace: "cadence"
+    cass-visibility:
+      nosql:
+        pluginName: "cassandra"
+        hosts: "127.0.0.1"
+        keyspace: "cadence_visibility"
+
+ringpop:
+  name: cadence
+  bootstrapMode: hosts
+  bootstrapHosts: [ "127.0.0.1:7933", "127.0.0.1:7934", "127.0.0.1:7935" ]
+  maxJoinDuration: 30s
+
+services:
+  frontend:
+    rpc:
+      port: 7933
+      grpcPort: 7833
+      bindOnLocalHost: true
+      grpcMaxMsgSize: 33554432
+    metrics:
+      statsd:
+        hostPort: "127.0.0.1:8125"
+        prefix: "cadence"
+    pprof:
+      port: 7936
+
+  matching:
+    rpc:
+      port: 7935
+      grpcPort: 7835
+      bindOnLocalHost: true
+      grpcMaxMsgSize: 33554432
+    metrics:
+      statsd:
+        hostPort: "127.0.0.1:8125"
+        prefix: "cadence"
+    pprof:
+      port: 7938
+
+  history:
+    rpc:
+      port: 7934
+      grpcPort: 7834
+      bindOnLocalHost: true
+      grpcMaxMsgSize: 33554432
+    metrics:
+      statsd:
+        hostPort: "127.0.0.1:8125"
+        prefix: "cadence"
+    pprof:
+      port: 7937
+
+  worker:
+    rpc:
+      port: 7939
+      bindOnLocalHost: true
+    metrics:
+      statsd:
+        hostPort: "127.0.0.1:8125"
+        prefix: "cadence"
+    pprof:
+      port: 7940
+
+clusterGroupMetadata:
+  failoverVersionIncrement: 10
+  primaryClusterName: "cluster0"
+  currentClusterName: "cluster0"
+  clusterGroup:
+    cluster0:
+      enabled: true
+      initialFailoverVersion: 0
+      newInitialFailoverVersion: 1 # migrating to this new failover version
+      rpcAddress: "localhost:7833" # this is to let worker service and XDC replicator connected to the frontend service. In cluster setup, localhost will not work
+      rpcTransport: "grpc"
+
+dcRedirectionPolicy:
+  policy: "noop"
+  toDC: ""
+
+archival:
+  history:
+    status: "enabled"
+    enableRead: true
+    provider:
+      filestore:
+        fileMode: "0666"
+        dirMode: "0766"
+      gstorage:
+        credentialsPath: "/tmp/gcloud/keyfile.json"
+  visibility:
+    status: "enabled"
+    enableRead: true
+    provider:
+      filestore:
+        fileMode: "0666"
+        dirMode: "0766"
+
+domainDefaults:
+  archival:
+    history:
+      status: "enabled"
+      URI: "file:///tmp/cadence_archival/development"
+    visibility:
+      status: "enabled"
+      URI: "file:///tmp/cadence_vis_archival/development"
+
+dynamicconfig:
+  client: filebased
+  configstore:
+    pollInterval: "10s"
+    updateRetryAttempts: 2
+    FetchTimeout: "2s"
+    UpdateTimeout: "2s"
+  filebased:
+    filepath: "config/dynamicconfig/development.yaml"
+    pollInterval: "10s"
+
+blobstore:
+  filestore:
+    outputDirectory: "/tmp/blobstore"
+
+asyncWorkflowQueues:
+  queue1:
+    type: "kafka"
+    config:
+      connection:
+        brokers:
+          - "localhost:9092"
+      topic: "async-wf-topic1"

docker/config_template.yaml

@@ -304,3 +304,14 @@ authorization:
         jwtCredentials:
             algorithm: "RS256"
             publicKey: {{ default .Env.OAUTH_PUBLIC_KEY "" }}
+
+{{- if .Env.ASYNC_WF_KAFKA_QUEUE_ENABLED }}
+asyncWorkflowQueues:
+  queue1:
+    type: "kafka"
+    config:
+      connection:
+        brokers:
+          - {{ default .Env.KAFKA_SEEDS "" }}:{{ default .Env.KAFKA_PORT "9092" }}
+      topic: {{ default .Env.ASYNC_WF_KAFKA_QUEUE_TOPIC "default-topic" }}
+{{- end }}

docker/docker-compose-async-wf-kafka.yml

@@ -0,0 +1,90 @@
+version: '3'
+services:
+  cassandra:
+    image: cassandra:4.1.1
+    ports:
+      - "9042:9042"
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus:/etc/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    ports:
+      - '9090:9090'
+  node-exporter:
+    image: prom/node-exporter
+    ports:
+      - '9100:9100'
+  cadence:
+    image: ubercadence/server:master-auto-setup-with-kafka
+    ports:
+     - "8000:8000"
+     - "8001:8001"
+     - "8002:8002"
+     - "8003:8003"
+     - "7933:7933"
+     - "7934:7934"
+     - "7935:7935"
+     - "7939:7939"
+     - "7833:7833"
+    environment:
+      - "ASYNC_WF_KAFKA_QUEUE_ENABLED=true"
+      - "ASYNC_WF_KAFKA_QUEUE_TOPIC=async-wf-topic1" # this should match the topic name in the config yml
+      - "KAFKA_SEEDS=kafka"
+      - "KAFKA_PORT=9092"
+      - "ZOOKEEPER_SEEDS=zookeeper"
+      - "ZOOKEEPER_PORT=2181"
+      - "CASSANDRA_SEEDS=cassandra"
+      - "PROMETHEUS_ENDPOINT_0=0.0.0.0:8000"
+      - "PROMETHEUS_ENDPOINT_1=0.0.0.0:8001"
+      - "PROMETHEUS_ENDPOINT_2=0.0.0.0:8002"
+      - "PROMETHEUS_ENDPOINT_3=0.0.0.0:8003"
+      - "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml"
+      - "LOG_LEVEL=debug"
+    depends_on:
+      cassandra:
+        condition: service_healthy
+      prometheus:
+        condition: service_started
+      kafka:
+        condition: service_started
+  cadence-web:
+    image: ubercadence/web:latest
+    environment:
+      - "CADENCE_TCHANNEL_PEERS=cadence:7933"
+    ports:
+      - "8088:8088"
+    depends_on:
+      - cadence
+  grafana:
+    image: grafana/grafana
+    volumes:
+      - ./grafana:/etc/grafana
+    user: "1000"
+    depends_on:
+      - prometheus
+    ports:
+      - '3000:3000'
+  zookeeper:
+    image: wurstmeister/zookeeper:3.4.6
+    ports:
+      - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka:2.12-2.1.1
+    depends_on:
+      - zookeeper
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

docker/start.sh

@@ -155,9 +155,48 @@ wait_for_db() {
     fi
 }
 
+wait_for_async_wf_queue_kafka() {
+    ready="false"
+    while [ "$ready" != "true" ]; do
+        brokers=$(echo dump | nc "$ZOOKEEPER_SEEDS" "$ZOOKEEPER_PORT" | grep brokers | wc -l)
+        if [ "$brokers" -gt 0 ]; then
+            ready="true"
+        else
+            echo 'waiting for kafka broker to show up in zookeeper'
+            sleep 3
+        fi
+    done
+
+    echo 'kafka broker started'
+}
+
+setup_async_wf_queue() {
+    if [ "$ASYNC_WF_KAFKA_QUEUE_ENABLED" != "true" ]; then
+        return
+    fi
+
+    wait_for_async_wf_queue_kafka
+
+    sh $KAFKA_HOME/bin/kafka-topics.sh --create --if-not-exists \
+            --zookeeper "$ZOOKEEPER_SEEDS:$ZOOKEEPER_PORT"  \
+            --topic "$ASYNC_WF_KAFKA_QUEUE_TOPIC" \
+            --partitions 10 \
+            --replication-factor 1
+
+    created=$(sh $KAFKA_HOME/bin/kafka-topics.sh --describe --topic $ASYNC_WF_KAFKA_QUEUE_TOPIC --zookeeper zookeeper:2181 | grep "Topic:$ASYNC_WF_KAFKA_QUEUE_TOPIC")
+    if [ -z "$created" ]; then
+        echo 'kafka topic is not created'
+        exit 1
+    fi
+
+    echo "Kafka topic $ASYNC_WF_KAFKA_QUEUE_TOPIC for async workflows created"
+}
+
 wait_for_db
 if [ "$SKIP_SCHEMA_SETUP" != true ]; then
     setup_schema
 fi
 
+setup_async_wf_queue
+
 exec /start-cadence.sh

scripts/buildkite/docker-build.sh

@@ -6,6 +6,7 @@ echo "Building docker images for $BUILDKITE_MESSAGE"
 
 docker build . -f Dockerfile -t ubercadence/server:master --build-arg TARGET=server
 docker build . -f Dockerfile -t ubercadence/server:master-auto-setup --build-arg TARGET=auto-setup
+docker build . -f Dockerfile -t ubercadence/server:master-auto-setup-with-kafka --build-arg TARGET=auto-setup-with-kafka
 docker build . -f Dockerfile -t ubercadence/cli:master --build-arg TARGET=cli
 docker build . -f Dockerfile -t ubercadence/cadence-bench:master --build-arg TARGET=bench
 docker build . -f Dockerfile -t ubercadence/cadence-canary:master --build-arg TARGET=canary