Use ElasticMQ instead of fake_sqs

changelog.d/0-release-notes/elasticmq

@@ -0,0 +1,8 @@
+* Replace fake-sqs with ElasticMQ
+
+ElasticMQ is an actively maintained project, fake-sqs hasn't seen a commit since
+2018. This is not expected to have any noticeable effect on deployments that
+don't have any extra configurations for the SQS queues. If the fake-aws-sqs
+chart had configured custom queue names, they have couple of extra limitations:
+- The queue names must only contain alphanumeric characters and hyphens.
+- The FIFO queue names must end in `.fifo`.

charts/fake-aws-sqs/templates/configmap.yaml

@@ -0,0 +1,60 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "fullname" . }}
+  labels:
+    app: {{ template "fullname" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+data:
+  elasticmq.conf: |
+    include classpath("application.conf")
+
+    # What is the outside visible address of this ElasticMQ node
+    # Used to create the queue URL (may be different from bind address!)
+    node-address {
+      protocol = http
+      host = localhost
+      port = {{ .Values.service.httpPort }}
+      context-path = ""
+    }
+
+    rest-sqs {
+      enabled = true
+      bind-port = {{ .Values.service.httpPort }}
+      bind-hostname = "0.0.0.0"
+      # Possible values: relaxed, strict
+      sqs-limits = strict
+    }
+
+    rest-stats {
+      enabled = true
+      bind-port = 9325
+      bind-hostname = "0.0.0.0"
+    }
+
+    # Should the node-address be generated from the bind port/hostname
+    # Set this to true e.g. when assigning port automatically by using port 0.
+    generate-node-address = false
+
+    queues {
+    {{- range $i, $queueName := .Values.queueNames }}
+      "{{ $queueName }}" {
+      {{- if hasSuffix ".fifo" $queueName }}
+        fifo = true
+      {{- end }}
+      }
+    {{- end }}
+    }
+
+    messages-storage {
+      enabled = true
+      uri = "jdbc:h2:/data/elasticmq.db"
+    }
+
+    # Region and accountId which will be included in resource ids
+    aws {
+      region = eu-west-1
+      accountId = 000000000000
+    }

charts/fake-aws-sqs/templates/deployment.yaml

@@ -16,6 +16,8 @@ spec:
     metadata:
       labels:
         app: {{ template "fullname" . }}
+      annotations:
+        checksum/configmap: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum }}
     spec:
       containers:
       - name: fake-aws-sqs
@@ -24,45 +26,22 @@ spec:
         - containerPort: {{ .Values.service.httpPort }}
           name: http
           protocol: TCP
+        command:
+        - /sbin/tini
+        - --
+        - /opt/elasticmq/bin/elasticmq-native-server
+        - -Dconfig.file=/config/elasticmq.conf
+        - -Dlogback.configurationFile=/opt/logback.xml
         volumeMounts:
         - name: storage
           mountPath: /data
+        - name: config
+          mountPath: /config
         resources:
 {{ toYaml .Values.resources | indent 12 }}
-      - name: initiate-fake-aws-sqs
-        image: mesosphere/aws-cli:1.14.5
-        command: [/bin/sh]
-        args:
-        - -c
-        - |
-          exec_until_ready() {
-              until $1; do echo 'service not ready yet'; sleep 1; done
-          }
-          queue_exists() {
-              # NOTE: we use the '"' to match the queue name more exactly (otherwise there is some overlap)
-              OUTPUT=$(aws --endpoint-url=http://localhost:{{ $.Values.service.httpPort }} sqs list-queues | grep $1'"' | wc -l)
-              echo $OUTPUT
-          }
-
-          echo 'Creating AWS resources'
-          aws configure set aws_access_key_id dummy
-          aws configure set aws_secret_access_key dummy
-          aws configure set region eu-west-1
-
-          while true
-          do
-              # Recreate resources if needed
-              {{ range $i, $queueName := .Values.queueNames }}
-                  QUEUE=$(queue_exists "{{ $queueName }}")
-                  if [ "$QUEUE" == "1" ]
-                    then echo "Queue {{ $queueName }} exists, no need to re-create"
-                    else exec_until_ready "aws --endpoint-url=http://localhost:{{ $.Values.service.httpPort }} sqs create-queue --queue-name {{ $queueName }}"
-                  fi
-              {{ end }}
-
-              echo 'Sleeping 10'
-              sleep 10
-          done
       volumes:
         - emptyDir: {}
           name: "storage"
+        - name: config
+          configMap:
+            name: {{ template "fullname" . }}

charts/fake-aws-sqs/values.yaml

@@ -1,6 +1,6 @@
 image:
-  repository: airdock/fake-sqs
-  tag: 0.3.1
+  repository: softwaremill/elasticmq-native
+  tag: 1.5.2
 
 # TODO: in a wire-server chart, these queue names should match the ones defined in galley/brig/gundeck (i.e. only be defined once)
 queueNames:

charts/integration/templates/integration-integration.yaml

@@ -113,7 +113,6 @@ spec:
         # FUTUREWORK: Do all of this in the integration test binary
         integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }}
         integration-dynamic-backends-brig-index.sh --elasticsearch-server http://{{ .Values.config.elasticsearch.host }}:9200
-        integration-dynamic-backends-sqs.sh {{ .Values.config.sqsEndpointUrl }}
         integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }}
         integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }}
         {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }}

deploy/dockerephemeral/docker-compose.yaml

@@ -34,10 +34,12 @@ services:
 
   fake_sqs:
     container_name: demo_wire_sqs
-#    image: airdock/fake-sqs:0.3.1
-    image: julialongtin/airdock_fakesqs:0.0.9
+    image: softwaremill/elasticmq-native:1.5.2
     ports:
       - 127.0.0.1:4568:4568
+      - 127.0.0.1:9325:9325
+    volumes:
+      - ./docker/elasticmq.conf:/opt/elasticmq.conf
     networks:
       - demo_wire
 

deploy/dockerephemeral/docker/elasticmq.conf

@@ -0,0 +1,80 @@
+include classpath("application.conf")
+
+# What is the outside visible address of this ElasticMQ node
+# Used to create the queue URL (may be different from bind address!)
+node-address {
+  protocol = http
+  host = localhost
+  port = 4568
+  context-path = ""
+}
+
+rest-sqs {
+  enabled = true
+  bind-port = 4568
+  bind-hostname = "0.0.0.0"
+  # Possible values: relaxed, strict
+  sqs-limits = strict
+}
+
+rest-stats {
+  enabled = true
+  bind-port = 9325
+  bind-hostname = "0.0.0.0"
+}
+
+# Should the node-address be generated from the bind port/hostname
+# Set this to true e.g. when assigning port automatically by using port 0.
+generate-node-address = false
+
+queues {
+  default-queue-template {
+    defaultVisibilityTimeout = 1s
+  }
+
+  fifo-queue-template {
+    defaultVisibilityTimeout = 1s
+    fifo = true
+  }
+
+  integration-brig-events =  ${queues.default-queue-template}
+  integration-brig-events2 = ${queues.default-queue-template}
+  integration-brig-events3 = ${queues.default-queue-template}
+  integration-brig-events4 = ${queues.default-queue-template}
+  integration-brig-events5 = ${queues.default-queue-template}
+  integration-brig-events-federation-v0 = ${queues.default-queue-template}
+
+  integration-brig-events-internal =  ${queues.default-queue-template}
+  integration-brig-events-internal2 = ${queues.default-queue-template}
+  integration-brig-events-internal3 = ${queues.default-queue-template}
+  integration-brig-events-internal4 = ${queues.default-queue-template}
+  integration-brig-events-internal5 = ${queues.default-queue-template}
+  integration-brig-events-internal-federation-v0 = ${queues.default-queue-template}
+
+  "integration-user-events.fifo" = ${queues.fifo-queue-template}
+  "integration-user-events2.fifo" = ${queues.fifo-queue-template}
+  "integration-user-events3.fifo" = ${queues.fifo-queue-template}
+  "integration-user-events4.fifo" = ${queues.fifo-queue-template}
+  "integration-user-events5.fifo" = ${queues.fifo-queue-template}
+  "integration-user-events-federation-v0.fifo" = ${queues.fifo-queue-template}
+
+  integration-gundeck-events = ${queues.default-queue-template}
+  integration-gundeck-events2 = ${queues.default-queue-template}
+  integration-gundeck-events3 = ${queues.default-queue-template}
+  integration-gundeck-events4 = ${queues.default-queue-template}
+  integration-gundeck-events5 = ${queues.default-queue-template}
+  integration-gundeck-events-federation-v0 = ${queues.default-queue-template}
+
+  "integration-team-events.fifo" = ${queues.fifo-queue-template}
+  "integration-team-events2.fifo" = ${queues.fifo-queue-template}
+  "integration-team-events3.fifo" = ${queues.fifo-queue-template}
+  "integration-team-events4.fifo" = ${queues.fifo-queue-template}
+  "integration-team-events5.fifo" = ${queues.fifo-queue-template}
+  "integration-team-events-federation-v0.fifo" = ${queues.fifo-queue-template}
+}
+
+# Region and accountId which will be included in resource ids
+aws {
+  region = eu-west-1
+  accountId = 000000000000
+}

deploy/dockerephemeral/federation-v0/brig.yaml

@@ -36,7 +36,7 @@ federatorInternal:
 
 # You can set up local SQS/Dynamo running e.g. `../../deploy/dockerephemeral/run.sh`
 aws:
-  userJournalQueue: integration-user-events.fifo-federation-v0
+  userJournalQueue: integration-user-events-federation-v0.fifo
   # ^ Comment this out if you don't want to journal user events
   prekeyTable: integration-brig-prekeys-federation-v0
   sqsEndpoint: http://fake_sqs:4568 # https://sqs.eu-west-1.amazonaws.com

deploy/dockerephemeral/federation-v0/galley.yaml

@@ -88,6 +88,6 @@ logLevel: Warn
 logNetStrings: false
 
 journal: # if set, journals; if not set, disables journaling
-  queueName: integration-team-events.fifo-federation-v0
+  queueName: integration-team-events-federation-v0.fifo
   endpoint: http://demo_wire_sqs:4568 # https://sqs.eu-west-1.amazonaws.com
   region: eu-west-1

deploy/dockerephemeral/init.sh

@@ -22,25 +22,10 @@ for suffix in "" "2" "3" "4" "5" "-federation-v0"; do
     aws --endpoint-url=http://dynamodb:8000 dynamodb delete-table --table-name integration-brig-userkey-blacklist$suffix || true
     aws --endpoint-url=http://dynamodb:8000 dynamodb delete-table --table-name integration-brig-prekeys$suffix || true
 
-    # Create Dynamo/SQS resources
+    # Create Dynamo resources
     exec_until_ready "aws --endpoint-url=http://dynamodb:8000 dynamodb create-table --table-name integration-brig-userkey-blacklist$suffix --attribute-definitions AttributeName=key,AttributeType=S --key-schema AttributeName=key,KeyType=HASH --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5"
     exec_until_ready "aws --endpoint-url=http://dynamodb:8000 dynamodb create-table --table-name integration-brig-prekeys$suffix --attribute-definitions AttributeName=client,AttributeType=S --key-schema AttributeName=client,KeyType=HASH --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5"
 
-    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs create-queue --queue-name integration-brig-events$suffix"
-    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs set-queue-attributes --queue-url http://sqs:4568/integration-brig-events$suffix --attributes VisibilityTimeout=1"
-    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs create-queue --queue-name integration-brig-events-internal$suffix"
-    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs set-queue-attributes --queue-url http://sqs:4568/integration-brig-events-internal$suffix --attributes VisibilityTimeout=1"
-    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs create-queue --queue-name integration-user-events.fifo$suffix"
-    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs set-queue-attributes --queue-url http://sqs:4568/integration-user-events.fifo$suffix --attributes VisibilityTimeout=1"
-
-    # Gundeck's feedback queue
-    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs create-queue --queue-name integration-gundeck-events$suffix"
-    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs set-queue-attributes --queue-url http://sqs:4568/integration-gundeck-events$suffix --attributes VisibilityTimeout=1"
-
-    # Galley's team event queue
-    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs create-queue --queue-name integration-team-events.fifo$suffix"
-    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs set-queue-attributes --queue-url http://sqs:4568/integration-team-events.fifo$suffix --attributes VisibilityTimeout=1"
-
     # Verify sender's email address (ensure the sender address is in sync with the config in brig)
     exec_until_ready "aws --endpoint-url=http://ses:4579 ses verify-email-identity --email-address backend-integration$suffix@wire.com"
 
@@ -53,6 +38,13 @@ for suffix in "" "2" "3" "4" "5" "-federation-v0"; do
     # TODO: Lifecycle configuration for the bucket, if supported.
     aws --endpoint-url=http://s3:9000 s3api create-bucket --bucket "dummy-bucket$suffix"
     aws --endpoint-url=http://s3:9000 s3api wait bucket-exists --bucket "dummy-bucket$suffix"
+
+    # Check that SQS resources are created
+    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs get-queue-url --queue-name integration-brig-events$suffix"
+    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs get-queue-url --queue-name integration-brig-events-internal$suffix"
+    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs get-queue-url --queue-name integration-user-events$suffix.fifo"
+    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs get-queue-url --queue-name integration-gundeck-events"
+    exec_until_ready "aws --endpoint-url=http://sqs:4568 sqs get-queue-url --queue-name integration-team-events$suffix.fifo"
 done
 
 echo 'AWS resources created successfully!'

hack/helm_vars/fake-aws/values.yaml

@@ -1,3 +1,29 @@
 fake-aws-ses:
   enabled: true
   sesSender: "backend-integrationk8s@wire.com"
+
+fake-aws-sqs:
+  queueNames:
+  - "integration-brig-events"
+  - "integration-brig-events-internal"
+  - "integration-gundeck-events"
+  - "integration-user-events.fifo"
+  - "integration-team-events.fifo"
+
+  - "integration-brig-events3"
+  - "integration-brig-events-internal3"
+  - "integration-gundeck-events3"
+  - "integration-user-events3.fifo"
+  - "integration-team-events3.fifo"
+
+  - "integration-brig-events4"
+  - "integration-brig-events-internal4"
+  - "integration-gundeck-events4"
+  - "integration-user-events4.fifo"
+  - "integration-team-events4.fifo"
+
+  - "integration-brig-events5"
+  - "integration-brig-events-internal5"
+  - "integration-gundeck-events5"
+  - "integration-user-events5.fifo"
+  - "integration-team-events5.fifo"

integration/test/Testlib/ResourcePool.hs

@@ -94,14 +94,14 @@ backendResources dynConfs =
                     berFederatorInternal = Ports.portForDyn (Ports.ServiceInternal FederatorInternal) i,
                     berFederatorExternal = dynConf.federatorExternalPort,
                     berDomain = dynConf.domain,
-                    berAwsUserJournalQueue = "integration-user-events.fifo" <> suffix i,
+                    berAwsUserJournalQueue = "integration-user-events" <> suffix i <> ".fifo",
                     berAwsPrekeyTable = "integration-brig-prekeys" <> suffix i,
                     berAwsS3Bucket = "dummy-bucket" <> suffix i,
                     berAwsQueueName = "integration-gundeck-events" <> suffix i,
                     berBrigInternalEvents = "integration-brig-events-internal" <> suffix i,
                     berEmailSMSSesQueue = "integration-brig-events" <> suffix i,
                     berEmailSMSEmailSender = "backend-integration" <> suffix i <> "@wire.com",
-                    berGalleyJournal = "integration-team-events.fifo" <> suffix i,
+                    berGalleyJournal = "integration-team-events" <> suffix i <> ".fifo",
                     berVHost = dynConf.domain,
                     berNginzSslPort = Ports.portForDyn Ports.NginzSSL i,
                     berNginzHttp2Port = Ports.portForDyn Ports.NginzHttp2 i,
@@ -151,14 +151,14 @@ backendB =
       berFederatorInternal = Ports.port (Ports.ServiceInternal FederatorInternal) BackendB,
       berFederatorExternal = Ports.port Ports.FederatorExternal BackendB,
       berDomain = "b.example.com",
-      berAwsUserJournalQueue = "integration-user-events.fifo2",
+      berAwsUserJournalQueue = "integration-user-events2.fifo",
       berAwsPrekeyTable = "integration-brig-prekeys2",
       berAwsS3Bucket = "dummy-bucket2",
       berAwsQueueName = "integration-gundeck-events2",
       berBrigInternalEvents = "integration-brig-events-internal2",
       berEmailSMSSesQueue = "integration-brig-events2",
       berEmailSMSEmailSender = "backend-integration2@wire.com",
-      berGalleyJournal = "integration-team-events.fifo2",
+      berGalleyJournal = "integration-team-events2.fifo",
       -- FUTUREWORK: set up vhosts in dev/ci for example.com and b.example.com
       -- in case we want backendA and backendB to federate with a third backend
       -- (because otherwise both queues will overlap)