Fix kind setup for running end-to-end federation tests locally

Makefile

@@ -438,6 +438,14 @@ kind-delete:
 .PHONY: kind-reset
 kind-reset: kind-delete kind-cluster
 
+.PHONY: kind-upload-images
+kind-upload-images:
+	DOCKER_TAG=$(DOCKER_TAG) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) ./hack/bin/kind-upload-images.sh
+
+.PHONY: kind-upload-image
+kind-upload-image-%:
+	DOCKER_TAG=$(DOCKER_TAG) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) ./hack/bin/kind-upload-image.sh wireServer.imagesUnoptimizedNoDocs.$(*)
+
 .local/kind-kubeconfig:
 	mkdir -p $(CURDIR)/.local
 	kind get kubeconfig --name $(KIND_CLUSTER_NAME) > $(CURDIR)/.local/kind-kubeconfig

changelog.d/5-internal/kind-fix

@@ -0,0 +1 @@
+Fix kind setup for running end-to-end federation tests locally.

docs/src/developer/developer/how-to.md

@@ -112,21 +112,19 @@ This can be useful to get quicker feedback while working on multi-backend code o
 
 FUTUREWORK: this process is in development (update this section after it's confirmed to work):
 
-##### (i) Build images
+##### Run tests in kind
 
-(FUTUREWORK: implement a convenient shortcut to build images without actually uploading them also)
-```
-make upload-images-dev
-```
+0. Create a local kind cluster with `make kind-cluster`
+1. Upload images in docker-daemon running inside kind with `make kind-upload-images`
 
-##### (ii) Run tests in kind
+   *Note:* First time all the images need to be uploaded. When working on one
+   service it can be selectively uploaded using `make kind-upload-image-<name>`
+   (e.g. `make kind-upload-image-brig`).
+2. Install wire-server using `make kind-integration-setup`.
+3. Run tests using `make kind-integration-test`.
+4. Run end2end integration tests: `make kind-integration-e2e`.
 
-0. Create a local kind cluster with `make kind-cluster`
-1. Install wire-server using `make kind-integration-setup`.
-2. Run tests using `make kind-integration-test`.
-3. Run end2end integration tests: `make kind-integration-e2e`.
 
-* Implement re-tagging development tags as your user tag?
 
 #### 2.4 Deploy your local code to a kubernetes cluster
 

hack/bin/kind-upload-image.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# This script builds all the images in wireServer.images attribute of
+# $ROOT_DIR/nix/default.nix and uploads them to the docker registry using the
+# repository name specified in the image derivation and tag specified by
+# environment variable "$DOCKER_TAG".
+#
+# If $DOCKER_USER and $DOCKER_PASSWORD are provided, the script will use them to
+# upload the images.
+#
+# This script is intended to be run by CI/CD pipelines.
+
+set -euo pipefail
+
+set -x
+
+# nix attribute under wireServer from "$ROOT_DIR/nix" containing all the images
+readonly IMAGE_ATTR=${1:?$usage}
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+ROOT_DIR=$(cd -- "$SCRIPT_DIR/../../" &>/dev/null && pwd)
+readonly SCRIPT_DIR ROOT_DIR
+
+tmp_link_store=$(mktemp -d)
+
+image_stream_file="$tmp_link_store/image-stream"
+nix -v --show-trace -L build -f "$ROOT_DIR/nix" "$IMAGE_ATTR" -o "$image_stream_file"
+image_file="$tmp_link_store/image"
+image_file_tagged="$tmp_link_store/image-tagged"
+"$image_stream_file" > "$image_file"
+repo=$(skopeo list-tags "docker-archive://$image_file" | jq -r '.Tags[0] | split(":") | .[0]')
+skopeo copy --insecure-policy --additional-tag "$repo:$DOCKER_TAG" "docker-archive://$image_file" "docker-archive://$image_file_tagged"
+kind load image-archive "$image_file_tagged" --name "$KIND_CLUSTER_NAME"

hack/bin/kind-upload-images.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# This script builds all the images in wireServer.images attribute of
+# $ROOT_DIR/nix/default.nix and uploads them to the docker registry using the
+# repository name specified in the image derivation and tag specified by
+# environment variable "$DOCKER_TAG".
+#
+# If $DOCKER_USER and $DOCKER_PASSWORD are provided, the script will use them to
+# upload the images.
+#
+# This script is intended to be run by CI/CD pipelines.
+
+set -euo pipefail
+
+set -x
+
+# nix attribute under wireServer from "$ROOT_DIR/nix" containing all the images
+readonly IMAGES_ATTR="imagesUnoptimizedNoDocs"
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+ROOT_DIR=$(cd -- "$SCRIPT_DIR/../../" &>/dev/null && pwd)
+readonly SCRIPT_DIR ROOT_DIR
+
+tmp_link_store=$(mktemp -d)
+image_list_file="$tmp_link_store/image-list"
+nix -v --show-trace -L build -f "$ROOT_DIR/nix" wireServer.imagesList -o "$image_list_file"
+
+xargs -I {} -P 10 "$SCRIPT_DIR/kind-upload-image.sh" "wireServer.$IMAGES_ATTR.{}" < "$image_list_file"
+
+for image_name in nginz nginz-disco; do
+    printf '*** Unploading image %s\n' "$image_name"
+    "$SCRIPT_DIR/kind-upload-image.sh" "$image_name"
+done

hack/helm_vars/redis-cluster/values.yaml → hack/helm_vars/redis-cluster/values.yaml.gotmpl

@@ -1,5 +1,5 @@
 global:
-  storageClass: csi-hostpath-sc
+  storageClass: {{ .Values.redisStorageClass }}
 
 redis-cluster:
   persistence:

hack/helmfile-single.yaml

@@ -37,7 +37,7 @@ releases:
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/redis-cluster'
     values:
-      - './helm_vars/redis-cluster/values.yaml'
+      - './helm_vars/redis-cluster/values.yaml.gotmpl'
 
   - name: '{{ .Values.namespace }}-nginx-ingress-controller'
     namespace: '{{ .Values.namespace }}'

hack/helmfile.yaml

@@ -19,13 +19,15 @@ environments:
       - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }}
       - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
       - imagePullPolicy: Always
+      - redisStorageClass: csi-hostpath-sc
   kind:
     values:
       - namespace: {{ requiredEnv "NAMESPACE_1" }}
       - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }}
       - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }}
       - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
       - imagePullPolicy: Never
+      - redisStorageClass: standard
 
 repositories:
   - name: stable
@@ -59,13 +61,13 @@ releases:
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/redis-cluster'
     values:
-      - './helm_vars/redis-cluster/values.yaml'
+      - './helm_vars/redis-cluster/values.yaml.gotmpl'
 
   - name: '{{ .Values.namespace }}-redis-cluster-2'
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/redis-cluster'
     values:
-      - './helm_vars/redis-cluster/values.yaml'
+      - './helm_vars/redis-cluster/values.yaml.gotmpl'
 
   - name: '{{ .Values.namespace }}-nginx-ingress-controller'
     namespace: '{{ .Values.namespace }}'

services/brig/federation-tests.sh

@@ -36,5 +36,12 @@ while read -r ip; do
   alsoProxyOptions+=("--also-proxy=${ip}")
 done < <(kubectl get pods -n "$NAMESPACE" -l app=cannon -o json | jq -r '.items[].status.podIPs[].ip')
 
+AWS_ACCESS_KEY_ID="$(kubectl get secret -n "$NAMESPACE" brig -o json | jq -r '.data | map_values(@base64d) | .awsKeyId')"
+export AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY="$(kubectl get secret -n "$NAMESPACE" brig -o json | jq -r '.data | map_values(@base64d) | .awsSecretKey')"
+export AWS_SECRET_ACCESS_KEY
+AWS_REGION="$(kubectl get deployment -n "$NAMESPACE" brig -o json | jq -r '.spec.template.spec.containers | map(.env | map(select(.name == "AWS_REGION").value))[0][0]')"
+export AWS_REGION
+
 # shellcheck disable=SC2086
 telepresence --namespace "$NAMESPACE" --also-proxy=cassandra-ephemeral ${alsoProxyOptions[*]} --run bash -c "export INTEGRATION_FEDERATION_TESTS=1; ./dist/brig-integration -p federation-end2end-user -i i.yaml -s b.yaml"