From 7114528aac2b8d1936f4d7f29a8acacdd85b231e Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 23 Jan 2023 11:15:09 +0100 Subject: [PATCH 1/6] Add scripts to load nix-built images into kind --- Makefile | 8 ++++++++ hack/bin/kind-upload-image.sh | 33 +++++++++++++++++++++++++++++++++ hack/bin/kind-upload-images.sh | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100755 hack/bin/kind-upload-image.sh create mode 100755 hack/bin/kind-upload-images.sh diff --git a/Makefile b/Makefile index affc632be21..7aa65a0e864 100644 --- a/Makefile +++ b/Makefile @@ -438,6 +438,14 @@ kind-delete: .PHONY: kind-reset kind-reset: kind-delete kind-cluster +.PHONY: kind-upload-images +kind-upload-images: + DOCKER_TAG=$(DOCKER_TAG) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) ./hack/bin/kind-upload-images.sh + +.PHONY: kind-upload-image +kind-upload-image-%: + DOCKER_TAG=$(DOCKER_TAG) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) ./hack/bin/kind-upload-image.sh wireServer.imagesUnoptimizedNoDocs.$(*) + .local/kind-kubeconfig: mkdir -p $(CURDIR)/.local kind get kubeconfig --name $(KIND_CLUSTER_NAME) > $(CURDIR)/.local/kind-kubeconfig diff --git a/hack/bin/kind-upload-image.sh b/hack/bin/kind-upload-image.sh new file mode 100755 index 00000000000..096280a0911 --- /dev/null +++ b/hack/bin/kind-upload-image.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# This script builds all the images in wireServer.images attribute of +# $ROOT_DIR/nix/default.nix and uploads them to the docker registry using the +# repository name specified in the image derivation and tag specified by +# environment variable "$DOCKER_TAG". +# +# If $DOCKER_USER and $DOCKER_PASSWORD are provided, the script will use them to +# upload the images. +# +# This script is intended to be run by CI/CD pipelines. + +set -euo pipefail + +set -x + +# nix attribute under wireServer from "$ROOT_DIR/nix" containing all the images +readonly IMAGE_ATTR=${1:?$usage} + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +ROOT_DIR=$(cd -- "$SCRIPT_DIR/../../" &>/dev/null && pwd) +readonly SCRIPT_DIR ROOT_DIR + +tmp_link_store=$(mktemp -d) + +image_stream_file="$tmp_link_store/image-stream" +nix -v --show-trace -L build -f "$ROOT_DIR/nix" "$IMAGE_ATTR" -o "$image_stream_file" +image_file="$tmp_link_store/image" +image_file_tagged="$tmp_link_store/image-tagged" +"$image_stream_file" > "$image_file" +repo=$(skopeo list-tags "docker-archive://$image_file" | jq -r '.Tags[0] | split(":") | .[0]') +skopeo copy --additional-tag "$repo:$DOCKER_TAG" "docker-archive://$image_file" "docker-archive://$image_file_tagged" +kind load image-archive "$image_file_tagged" --name "$KIND_CLUSTER_NAME" diff --git a/hack/bin/kind-upload-images.sh b/hack/bin/kind-upload-images.sh new file mode 100755 index 00000000000..5ba04b7025e --- /dev/null +++ b/hack/bin/kind-upload-images.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# This script builds all the images in wireServer.images attribute of +# $ROOT_DIR/nix/default.nix and uploads them to the docker registry using the +# repository name specified in the image derivation and tag specified by +# environment variable "$DOCKER_TAG". +# +# If $DOCKER_USER and $DOCKER_PASSWORD are provided, the script will use them to +# upload the images. +# +# This script is intended to be run by CI/CD pipelines. + +set -euo pipefail + +set -x + +# nix attribute under wireServer from "$ROOT_DIR/nix" containing all the images +readonly IMAGES_ATTR="imagesUnoptimizedNoDocs" + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +ROOT_DIR=$(cd -- "$SCRIPT_DIR/../../" &>/dev/null && pwd) +readonly SCRIPT_DIR ROOT_DIR + +tmp_link_store=$(mktemp -d) +image_list_file="$tmp_link_store/image-list" +nix -v --show-trace -L build -f "$ROOT_DIR/nix" wireServer.imagesList -o "$image_list_file" + +xargs -I {} -P 10 "$SCRIPT_DIR/kind-upload-image.sh" "wireServer.$IMAGES_ATTR.{}" < "$image_list_file" + +for image_name in nginz nginz-disco; do + printf '*** Unploading image %s\n' "$image_name" + "$SCRIPT_DIR/kind-upload-image.sh" "$image_name" +done From f03e248d1ffb8e42a149cbeb84154de742dbcc44 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 23 Jan 2023 11:16:17 +0100 Subject: [PATCH 2/6] Fix redis-cluster in kind setup --- .../redis-cluster/{values.yaml => values.yaml.gotmpl} | 2 +- hack/helmfile-single.yaml | 2 +- hack/helmfile.yaml | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) rename hack/helm_vars/redis-cluster/{values.yaml => values.yaml.gotmpl} (66%) diff --git a/hack/helm_vars/redis-cluster/values.yaml b/hack/helm_vars/redis-cluster/values.yaml.gotmpl similarity index 66% rename from hack/helm_vars/redis-cluster/values.yaml rename to hack/helm_vars/redis-cluster/values.yaml.gotmpl index 888fbc02627..5381d26cbdf 100644 --- a/hack/helm_vars/redis-cluster/values.yaml +++ b/hack/helm_vars/redis-cluster/values.yaml.gotmpl @@ -1,5 +1,5 @@ global: - storageClass: csi-hostpath-sc + storageClass: {{ .Values.redisStorageClass }} redis-cluster: persistence: diff --git a/hack/helmfile-single.yaml b/hack/helmfile-single.yaml index 790412bf71f..3a770ee1469 100644 --- a/hack/helmfile-single.yaml +++ b/hack/helmfile-single.yaml @@ -37,7 +37,7 @@ releases: namespace: '{{ .Values.namespace }}' chart: '../.local/charts/redis-cluster' values: - - './helm_vars/redis-cluster/values.yaml' + - './helm_vars/redis-cluster/values.yaml.gotmpl' - name: '{{ .Values.namespace }}-nginx-ingress-controller' namespace: '{{ .Values.namespace }}' diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index 825e49edff5..9dd863334cf 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -19,6 +19,7 @@ environments: - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }} - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }} - imagePullPolicy: Always + - redisStorageClass: csi-hostpath-sc kind: values: - namespace: {{ requiredEnv "NAMESPACE_1" }} @@ -26,6 +27,7 @@ environments: - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }} - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }} - imagePullPolicy: Never + - redisStorageClass: standard repositories: - name: stable @@ -59,13 +61,13 @@ releases: namespace: '{{ .Values.namespace }}' chart: '../.local/charts/redis-cluster' values: - - './helm_vars/redis-cluster/values.yaml' + - './helm_vars/redis-cluster/values.yaml.gotmpl' - name: '{{ .Values.namespace }}-redis-cluster-2' namespace: '{{ .Values.namespaceFed2 }}' chart: '../.local/charts/redis-cluster' values: - - './helm_vars/redis-cluster/values.yaml' + - './helm_vars/redis-cluster/values.yaml.gotmpl' - name: '{{ .Values.namespace }}-nginx-ingress-controller' namespace: '{{ .Values.namespace }}' From e4701928ff63849fa39bc2986258d8692e4181ae Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 23 Jan 2023 11:20:30 +0100 Subject: [PATCH 3/6] Update how to use kind --- docs/src/developer/developer/how-to.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/src/developer/developer/how-to.md b/docs/src/developer/developer/how-to.md index b900fc4f1f9..68250bba21c 100644 --- a/docs/src/developer/developer/how-to.md +++ b/docs/src/developer/developer/how-to.md @@ -112,21 +112,19 @@ This can be useful to get quicker feedback while working on multi-backend code o FUTUREWORK: this process is in development (update this section after it's confirmed to work): -##### (i) Build images +##### Run tests in kind -(FUTUREWORK: implement a convenient shortcut to build images without actually uploading them also) -``` -make upload-images-dev -``` +0. Create a local kind cluster with `make kind-cluster` +1. Upload images in docker-daemon running inside kind with `make kind-upload-images` -##### (ii) Run tests in kind + *Note:* First time all the images need to be uploaded. When working on one + service it can be selectively uploaded using `make kind-upload-image-` + (e.g. `make kind-upload-image-brig`). +2. Install wire-server using `make kind-integration-setup`. +3. Run tests using `make kind-integration-test`. +4. Run end2end integration tests: `make kind-integration-e2e`. -0. Create a local kind cluster with `make kind-cluster` -1. Install wire-server using `make kind-integration-setup`. -2. Run tests using `make kind-integration-test`. -3. Run end2end integration tests: `make kind-integration-e2e`. -* Implement re-tagging development tags as your user tag? #### 2.4 Deploy your local code to a kubernetes cluster From 0cb42711bcc1b46d2366528b9243e75b71512430 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 23 Jan 2023 11:21:18 +0100 Subject: [PATCH 4/6] Changelog --- changelog.d/5-internal/kind-fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5-internal/kind-fix diff --git a/changelog.d/5-internal/kind-fix b/changelog.d/5-internal/kind-fix new file mode 100644 index 00000000000..2b7d9f1b3b8 --- /dev/null +++ b/changelog.d/5-internal/kind-fix @@ -0,0 +1 @@ +Fix kind setup for running end-to-end federation tests locally. \ No newline at end of file From d9d0fbbf5071e7f4ceffa971836a5f0aeea6f088 Mon Sep 17 00:00:00 2001 From: Stefan Berthold Date: Tue, 24 Jan 2023 13:40:55 +0000 Subject: [PATCH 5/6] allow skopeo insecure policy --- hack/bin/kind-upload-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/bin/kind-upload-image.sh b/hack/bin/kind-upload-image.sh index 096280a0911..61b24c7937f 100755 --- a/hack/bin/kind-upload-image.sh +++ b/hack/bin/kind-upload-image.sh @@ -29,5 +29,5 @@ image_file="$tmp_link_store/image" image_file_tagged="$tmp_link_store/image-tagged" "$image_stream_file" > "$image_file" repo=$(skopeo list-tags "docker-archive://$image_file" | jq -r '.Tags[0] | split(":") | .[0]') -skopeo copy --additional-tag "$repo:$DOCKER_TAG" "docker-archive://$image_file" "docker-archive://$image_file_tagged" +skopeo copy --insecure-policy --additional-tag "$repo:$DOCKER_TAG" "docker-archive://$image_file" "docker-archive://$image_file_tagged" kind load image-archive "$image_file_tagged" --name "$KIND_CLUSTER_NAME" From fe4843a2adf10a6a90b338d5ea7870ae18dca528 Mon Sep 17 00:00:00 2001 From: Stefan Berthold Date: Tue, 24 Jan 2023 13:41:39 +0000 Subject: [PATCH 6/6] fetch AWS secrets from Kubernetes --- services/brig/federation-tests.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/brig/federation-tests.sh b/services/brig/federation-tests.sh index 6212f75f853..5cea51393d9 100755 --- a/services/brig/federation-tests.sh +++ b/services/brig/federation-tests.sh @@ -36,5 +36,12 @@ while read -r ip; do alsoProxyOptions+=("--also-proxy=${ip}") done < <(kubectl get pods -n "$NAMESPACE" -l app=cannon -o json | jq -r '.items[].status.podIPs[].ip') +AWS_ACCESS_KEY_ID="$(kubectl get secret -n "$NAMESPACE" brig -o json | jq -r '.data | map_values(@base64d) | .awsKeyId')" +export AWS_ACCESS_KEY_ID +AWS_SECRET_ACCESS_KEY="$(kubectl get secret -n "$NAMESPACE" brig -o json | jq -r '.data | map_values(@base64d) | .awsSecretKey')" +export AWS_SECRET_ACCESS_KEY +AWS_REGION="$(kubectl get deployment -n "$NAMESPACE" brig -o json | jq -r '.spec.template.spec.containers | map(.env | map(select(.name == "AWS_REGION").value))[0][0]')" +export AWS_REGION + # shellcheck disable=SC2086 telepresence --namespace "$NAMESPACE" --also-proxy=cassandra-ephemeral ${alsoProxyOptions[*]} --run bash -c "export INTEGRATION_FEDERATION_TESTS=1; ./dist/brig-integration -p federation-end2end-user -i i.yaml -s b.yaml"