Add amqpprox performance testing tool

This commit adds a new test tool to the amqpprox repo. Extract from the
new `tests/performance_tester/README.md`:

```

`amqpprox` performance has been tested mainly in two ways:

1. Total data throughput achieved by connected clients (MB/s)
2. Total connection establishment throughput achieved (connections/s)

The performance tester in this folder helps with both of these by:

1. It contains a dummy AMQP 0.9.1. server which walks the AMQP handshake and ignores all future frames except close.
2. It can run parallel AMQP clients connecting to an amqpprox instance.
    1. Testing data throughput probably wants to send more, larger messages with fewer connections
    2. Testing connection throughput probably wants to send fewer, smaller messages with many connections.
```

I tried lapin instead of amiquip when investigating adding TLS
support for the `client -> amqpprox` hop, but found lapin performed
significantly worse than `amiquip` and often failed to shutdown cleanly.

It's possible this is because amiquip was given a dedicated thread and lapin
ran async, or perhaps too many async clients were started & failed to make
progress. But initial numbers were 75% down. I didn't think this would properly
exercise amqpprox so dropped it for now. It's possible we just need to
go down the dummy AMQP client route too.

My test setup results so far indicate a roughly 50% impact in connection
throughput when enabling TLS from amqpprox to the broker: 2000+
connections/s down to around ~1000/s. Overall data
throughput is affected significantly less, with a ~5% reduction: 690MB/s
-> 630MB/s. This test setup runs amqpprox on a different, nearby,
machine to amqpprox_perf_tester, on similar-ish spec machines to what we
use in production. Although I've mainly been looking at the difference
in performance here, the absolute numbers are interesting too. I am not
sure how we end up 'only' achieving 700MB/s, it's possible the test
client can't fully utilise amqpprox here.

This commit also upgrades the integration tests to rmq 3.7.28 dockerhub image
since 3.7.9 hasn't been built in over 3 years. I had upgraded to 3.9 but that
didn't pass the integration tests. I'd prefer to investigate that separately

Co-authored-by: Alaric <alaric@bloomberg.net>

.dockerignore

@@ -1 +1,2 @@
 build
+tests/performance_tester/target

Makefile

@@ -22,6 +22,10 @@ init:
 clean:
 	cd $(BUILDDIR) && make clean
 
+integration-tests:
+	python3.8 -mpytest -s tests/performance_tester/integration-tests.py
+	./tests/acceptance/run.sh
+
 DOCKER_IMAGE ?= amqpprox
 DOCKER_BUILDDIR ?= build/docker-$(DOCKER_IMAGE)
 DOCKER_ARGS ?= $(DOCKER_EXTRA_ARGS) -v $(CUR_DIR):/source -v $(CUR_DIR)/$(DOCKER_BUILDDIR):/build -it $(DOCKER_IMAGE)
@@ -50,7 +54,7 @@ docker-integration-tests: BUILD_FLAVOUR ?= conan
 docker-integration-tests: BUILD_DOCKERFILE ?= buildfiles/$(BUILD_FLAVOUR)/integration.Dockerfile
 docker-integration-tests:
 	docker build -t $(DOCKER_IMAGE) -f $(BUILD_DOCKERFILE) .
-	docker run $(DOCKER_IMAGE)
+	docker run $(DOCKER_IMAGE) make integration-tests
 
 docs:
 	doxygen Doxygen.config

buildfiles/conan/integration.Dockerfile

@@ -1,13 +1,16 @@
-FROM rabbitmq:3.7.9
+FROM rabbitmq:3.7.28
 
 ENV DEBIAN_FRONTEND=noninteractive
+
+# Install dependencies for integration tests
 RUN apt-get update && apt-get dist-upgrade -y --force-yes
 RUN apt-get install -y --force-yes python3.8 python3.8-distutils \
     curl llvm make cmake build-essential npm
 RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
 RUN python3.8 get-pip.py
-
-RUN python3.8 -m pip install setuptools conan robotframework pika amqp
+RUN python3.8 -m pip install setuptools conan robotframework pika amqp pytest
+ENV HOME="/root" PATH="/root/.cargo/bin:${PATH}"
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
 
 EXPOSE 15800
 EXPOSE 15801
@@ -21,6 +24,5 @@ RUN npm install
 WORKDIR /source
 
 RUN make setup && make init && make
-ENV ROBOT_SOURCE_DIR=/source/tests/acceptance
-ENV ROBOT_BINARY_DIR=/opt/rabbitmq/sbin
-ENTRYPOINT [ "/source/tests/acceptance/run.sh"]
+ENV ROBOT_SOURCE_DIR=/source/tests/acceptance ROBOT_BINARY_DIR=/opt/rabbitmq/sbin
+ENV AMQPPROX_BIN_DIR=/build/bin

tests/acceptance/connection.robot

@@ -36,14 +36,14 @@ Smoke suite setup
     ${SMOKE_PATH}=        Get Environment Variable   SMOKE_PATH
     ${WAIT_TIME}=         Get Environment Variable   WAIT_TIME
     ${LOG_CONSOLE}=       Get Environment Variable   LOG_CONSOLE
-    ${BUILD_PATH}=       Get Environment Variable   BUILD_PATH
+    ${AMQPPROX_BIN_DIR}=       Get Environment Variable   AMQPPROX_BIN_DIR
     Set suite variable    ${ROBOT_SOURCE_DIR}
     Set suite variable    ${BINARY_PATH}
     Set suite variable    ${SOURCE_PATH}
     Set suite variable    ${SMOKE_PATH}
     Set suite variable    ${WAIT_TIME}
     Set suite variable    ${LOG_CONSOLE}
-    Set suite variable    ${BUILD_PATH}
+    Set suite variable    ${AMQPPROX_BIN_DIR}
 
 Connection Test Setup
     Log  ""  console=yes

tests/acceptance/libs/AMQPProx.robot

@@ -20,8 +20,8 @@ Library         Process
 AMQPProx start
     Create Directory  /tmp/logs/amqpprox
     ${SOURCE_PATH}=       Get Environment Variable   SOURCE_PATH
-    ${BUILD_PATH}=       Get Environment Variable   BUILD_PATH
-    ${result}=  Start Process  ${BUILD_PATH}/amqpprox --cleanupIntervalMs 10 --controlSocket /tmp/amqpprox --logDirectory /tmp/logs/amqpprox
+    ${AMQPPROX_BIN_DIR}=  Get Environment Variable   AMQPPROX_BIN_DIR
+    ${result}=  Start Process  ${AMQPPROX_BIN_DIR}/amqpprox --cleanupIntervalMs 10 --controlSocket /tmp/amqpprox --logDirectory /tmp/logs/amqpprox
     ...                      shell=yes
     [Return]    ${result}
 

tests/acceptance/libs/AMQPProxCTL.robot

@@ -222,8 +222,8 @@ AMQPProxCTL VHOST force_disconnect
 AMQPProxCTL send command
     [Arguments]   @{arguments}
     ${SOURCE_PATH}=       Get Environment Variable   SOURCE_PATH
-    ${BUILD_PATH}=       Get Environment Variable   BUILD_PATH
-    ${result}=  Run Process  ${BUILD_PATH}/amqpprox_ctl
+    ${AMQPPROX_BIN_DIR}=  Get Environment Variable   AMQPPROX_BIN_DIR
+    ${result}=  Run Process  ${AMQPPROX_BIN_DIR}/amqpprox_ctl
     ...                      /tmp/amqpprox
     ...                      @{arguments}
     ...                      shell=yes

tests/acceptance/run.sh

@@ -16,7 +16,7 @@
 
 export BINARY_PATH=${ROBOT_BINARY_DIR:=/usr/bin}
 export SOURCE_PATH=${SOURCE_PATH:=/source}
-export BUILD_PATH=${BUILD_PATH:=/build/bin}
+export AMQPPROX_BIN_DIR=${AMQPPROX_BIN_DIR:=/build/bin}
 export ACCEPTANCE_PATH=${ACCEPTANCE_PATH:=/source/tests/acceptance}
 export SMOKE_PATH=${SMOKE_PATH:=/source/tests/acceptance/integration}
 export WAIT_TIME=${WAIT_TIME:=30}

tests/acceptance/smoke.robot

@@ -29,14 +29,14 @@ Smoke suite setup
     ${SMOKE_PATH}=        Get Environment Variable   SMOKE_PATH
     ${WAIT_TIME}=         Get Environment Variable   WAIT_TIME
     ${LOG_CONSOLE}=       Get Environment Variable   LOG_CONSOLE
-    ${BUILD_PATH}=       Get Environment Variable   BUILD_PATH
+    ${AMQPPROX_BIN_DIR}=  Get Environment Variable   AMQPPROX_BIN_DIR
     Set suite variable    ${ROBOT_SOURCE_DIR}
     Set suite variable    ${BINARY_PATH}
     Set suite variable    ${SOURCE_PATH}
     Set suite variable    ${SMOKE_PATH}
     Set suite variable    ${WAIT_TIME}
     Set suite variable    ${LOG_CONSOLE}
-    Set suite variable    ${BUILD_PATH}
+    Set suite variable    ${AMQPPROX_BIN_DIR}
 
 
 *** Test Cases ***
@@ -69,8 +69,8 @@ Smoke Test
     ...  console=${LOG_CONSOLE}
     ${result} =  Run Process  node
     ...                       ${SMOKE_PATH}/index.js
-    ...                       ${BUILD_PATH}/amqpprox
-    ...                       ${BUILD_PATH}/amqpprox_ctl
+    ...                       ${AMQPPROX_BIN_DIR}/amqpprox
+    ...                       ${AMQPPROX_BIN_DIR}/amqpprox_ctl
     ...                       ${WAIT_TIME}
     ...                       stdout=STDOUT
     ...                       stderr=STDOUT

tests/performance_tester/.gitignore

@@ -0,0 +1 @@
+target