Federation end2end test scripts and Makefile targets

Makefile

@@ -218,6 +218,7 @@ libzauth:
 # Run this again after changes to libraries or dependencies.
 .PHONY: hie.yaml
 hie.yaml:
+	stack build implicit-hie
 	stack exec gen-hie > hie.yaml
 
 #####################################
@@ -242,16 +243,30 @@ hie.yaml:
 #   - kubectl
 #   - a valid kubectl context configured (i.e. access to a kubernetes cluster)
 .PHONY: kube-integration
-kube-integration: charts-integration
+kube-integration: guard-tag charts-integration
 	# by default "test-<your computer username> is used as namespace
 	# you can override the default by setting the NAMESPACE environment variable
 	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup.sh
 	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-test.sh
 
+.PHONY: kube-integration-setup
+kube-integration-setup: guard-tag charts-integration
+	# by default "test-<your computer username> is used as namespace
+	# you can override the default by setting the NAMESPACE environment variable
+	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup.sh
+
 .PHONY: kube-integration-teardown
 kube-integration-teardown:
 	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-teardown.sh
 
+.PHONY: kube-integration-setup-federation
+kube-integration-setup-federation: guard-tag charts-integration
+	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup-federation.sh
+
+.PHONY: kube-integration-federation
+kube-integration-federation:
+	cd services/brig && ./federation-tests.sh $(NAMESPACE)
+
 .PHONY: latest-brig-tag
 latest-brig-tag:
 	./hack/bin/find-latest-docker-tag.sh
@@ -268,6 +283,12 @@ release-chart-%:
 	fi
 	make chart-$(*)
 
+.PHONY: guard-tag
+guard-tag:
+	@if [ "${DOCKER_TAG}" = "${USER}" ]; then \
+	      echo "Environment variable DOCKER_TAG not set to non-default value. Re-run with DOCKER_TAG=<something>. Try using 'make latest-brig-tag' for latest develop docker image tag";\
+	    exit 1; \
+	fi
 
 # Rationale for copying charts to a gitignored folder before modifying helm versions and docker image tags:
 #

charts/brig/templates/tests/brig-integration.yaml

@@ -53,7 +53,7 @@ spec:
     # same file-system.
     # The other test, "user.auth.cookies.limit", is skipped as it is flaky.
     # This is tracked in https://github.com/zinfra/backend-issues/issues/1150.
-    command: ["brig-integration", "--pattern", "!/turn/ && !/user.auth.cookies.limit/"]
+    command: ["brig-integration", "--pattern", "!/turn/ && !/user.auth.cookies.limit/ && !/brig-federation/"]
     volumeMounts:
     - name: "brig-integration"
       mountPath: "/etc/wire/integration"

docs/developer/dependencies.md

@@ -179,6 +179,42 @@ dependencies automatically - including `cryptobox-c`. If new system dependencies
 Just type `$ nix-shell` and you will automatically have `make`, `docker-compose` and `stack` in `PATH`.
 You can then run all the builds, and the native dependencies will be automatically present.
 
+## Telepresence
+
+You can instead use [telepresence](https://www.telepresence.io) to allow you to talk to services installed in a given kubernetes namespace on a local or remote kubernetes cluster using easy DNS names like: `curl http://elasticsearch:9200`.
+
+Requirements:
+
+* install telepresence (e.g. `nix-env -iA nixpkgs.telepresence`)
+* you need access to a kubernetes cluster
+* you need a namespace in which you have installed something (e.g. `make kube-integration-setup` will do this)
+
+### Telepresence example usage:
+
+```
+# terminal 1
+telepresence --namespace "$NAMESPACE" --also-proxy cassandra-ephemeral
+```
+
+```
+# terminal 2
+curl http://elasticsearch-ephemeral:9200
+```
+
+### Telepresence example usage 2:
+
+```
+# just one terminal
+telepresence --namespace "$NAMESPACE" --also-proxy cassandra-ephemeral --run bash -c "curl http://elasticsearch-ephemeral:9200"
+```
+
+### Telepresence usage discussion:
+
+* If you have `fake-aws` and `databases-ephemeral` helm charts set up, you can run either `brig` and other services locally (they connect to cassandra-inside-kubernetes)
+* If you also have `brig` and other haskell services running in kubernetes (e.g. you ran `make kube-integration-setup`, you can use telepresence to only run test executables (like `brig-integration`) locally which connect to services inside kubernetes.
+
+In both cases, you need to adjust the various integration configuration files and names so that this can work.
+
 ## Helm chart development, integration tests in kubernetes
 
 You need `kubectl`, `helm`, and a valid kubernetes context. Refer to https://docs.wire.com for details.

hack/bin/integration-setup-federation.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+USAGE="Usage: $0"
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TOP_LEVEL="$DIR/../.."
+
+export NAMESPACE=${NAMESPACE:-test-integration}
+
+$DIR/integration-setup.sh
+
+# The suffix '-fed2' must be kept in sync with configuration inside charts/brig/templates/tests/configmap.yaml
+export NAMESPACE=${NAMESPACE}-fed2
+
+$DIR/integration-setup.sh

hack/bin/integration-setup.sh

@@ -28,6 +28,8 @@ function printLogs() {
     kubectl -n ${NAMESPACE} get pods | grep Pending | awk '{print $1}' | xargs -n 1 -I{} bash -c "printf '\n\n----DESCRIBE 'pending' {}:\n'; kubectl -n ${NAMESPACE} describe pod {}" || true
 }
 
+FEDERATION_DOMAIN="$NAMESPACE.svc.cluster.local"
+
 for chart in "${charts[@]}"; do
     kubectl -n ${NAMESPACE} get pods
     valuesfile="${DIR}/../helm_vars/${chart}/values.yaml"
@@ -39,10 +41,12 @@ for chart in "${charts[@]}"; do
     # default is 5m but may not be enough on a fresh install including cassandra migrations
     TIMEOUT=10m
     set -x
-    helm upgrade --atomic --install --namespace "${NAMESPACE}" "${NAMESPACE}-${chart}" "${CHARTS_DIR}/${chart}" \
+    helm upgrade --install --namespace "${NAMESPACE}" "${NAMESPACE}-${chart}" "${CHARTS_DIR}/${chart}" \
         $option \
+        --set brig.config.optSettings.setFederationDomain="$FEDERATION_DOMAIN" \
+        --set galley.config.settings.federationDomain="$FEDERATION_DOMAIN" \
         --wait \
-        --timeout "$TIMEOUT" || printLogs
+        --timeout "$TIMEOUT"
     set +x
 done
 

hack/helm_vars/wire-server/values.yaml

@@ -224,3 +224,6 @@ spar:
       email: email:backend+spar@wire.com
     # Keep this in sync with setTeamInvitationTimeout in brig
     brigSettingsTeamInvitationTimeout: 10
+
+federator:
+  replicaCount: 1

services/brig/Makefile

@@ -86,15 +86,19 @@ $(DEB_INDEX): install
 
 .PHONY: i
 i:
-	INTEGRATION_USE_NGINZ=$(INTEGRATION_USE_NGINZ) ../integration.sh $(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml $(WIRE_INTEGRATION_TEST_OPTIONS)
+	INTEGRATION_USE_NGINZ=$(INTEGRATION_USE_NGINZ) ../integration.sh $(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml -p "!/brig-federation/" $(WIRE_INTEGRATION_TEST_OPTIONS)
 
 .PHONY: i-aws
 i-aws:
 	INTEGRATION_USE_REAL_AWS=1 INTEGRATION_USE_NGINZ=$(INTEGRATION_USE_NGINZ) ../integration.sh $(EXE_IT) -s $(NAME).integration-aws.yaml -i ../integration.yaml $(WIRE_INTEGRATION_TEST_OPTIONS)
 
+.PHONY: i-federation
+i-federation:
+	INTEGRATION_USE_NGINZ=$(INTEGRATION_USE_NGINZ) ../integration.sh $(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml -p brig-federation $(WIRE_INTEGRATION_TEST_OPTIONS)
+
 .PHONY: i-list
 i-list:
-	$(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml -l
+	../integration.sh $(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml -l
 
 i-%:
 	INTEGRATION_USE_NGINZ=$(INTEGRATION_USE_NGINZ) ../integration.sh $(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml -p "$*" $(WIRE_INTEGRATION_TEST_OPTIONS)

services/brig/federation-tests.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+USAGE="$0 <NAMESPACE>"
+NAMESPACE=${1:?$USAGE}
+
+# This script assumes:
+# * two wire-server backends under NAMEPACE and NAMESPACE-fed2 have been deployed with helm.
+# * you have a locally compiled brig-integration executable
+#
+# It then downloads the configmaps, performs a hacky override for two configuration flags,
+# and then uses telepresence to run a locally-compiled brig-integration executable against
+# the brigs and federators inside kubernetes in the two NAMESPACES.
+
+kubectl -n "$NAMESPACE" get configmap brig-integration -o jsonpath='{.data.integration\.yaml}' > i.yaml
+kubectl -n "$NAMESPACE" get configmap brig -o jsonpath='{.data.brig\.yaml}' > b.yaml
+
+# override some settings so the local brig-integration executable doesn't fail
+sed -i "s=privateKeys: /etc/wire/brig/secrets/secretkey.txt=privateKeys: test/resources/zauth/privkeys.txt=g" b.yaml
+sed -i "s=publicKeys: /etc/wire/brig/secrets/publickey.txt=publicKeys: test/resources/zauth/pubkeys.txt=g" b.yaml
+
+telepresence --namespace "$NAMESPACE" --also-proxy cassandra-ephemeral --run bash -c "./dist/brig-integration -p brig-federation -i i.yaml -s b.yaml"