Merge pull request #1360 from wireapp/release_2021_02_15

Release 2021-02-15

.gitignore

@@ -87,3 +87,8 @@ hie.yaml
 
 # generated files under .local
 .local
+
+# buildah-based docker image building
+.stack-root-buildah/
+.stack-work-buildah/
+dist-buildah

CHANGELOG.md

@@ -14,6 +14,42 @@
 
 -->
 
+# [2020-02-15]
+
+## Release Notes
+
+This release requires recreating brig's ES index *before* deployment. See [instructions](https://github.com/wireapp/wire-server/blob/e3064d101ef8e9074431049135d2319335de3117/docs/reference/elasticsearch-migration-2021-02-15.md).
+
+## Features
+
+* Team search: Add search by email (#1344) (#1286)
+* Add endpoint to get client metadata for many users (#1345)
+* Public end-point for getting the team size. (#1295)
+
+## Bug fixes and other updates
+
+* Wire.API.UserMap & Brig.API.Public: Fix Swagger docs (#1350)
+* Fix nix build on OSX (#1340)
+
+## Internal changes
+
+* [federation] Federation end2end test scripts and Makefile targets (#1341)
+* [federation] Brig integration tests (#1342)
+* Add stack 2.3.1 to shell.nix (#1347)
+* buildah: Use correct dist directory while building docker-images (#1352)
+* Add spar.scim_external table and follow changes (#1359)
+* buildah: Allow building only a given exec and fix brig templates (#1353)
+* Galley: Add /teams/:tid/members csv download (#1351) (#1351)
+* Faster local docker image building using buildah (#1349)
+* Replace federation guard with env var (#1346)
+* Update cassandra schema after latest changes (#1337)
+* Add fast-intermediate Dockerfile for faster PR CI (#1328)
+* dns-util: Allow running lookup with a given resolver (#1338)
+* Add missing internal qa routes (#1336)
+* Extract and rename PolyLog to a library for reusability (#1329)
+* Fix: Spar integration tests misconfigured on CI (#1343)
+
+
 # [2020-01-15]
 
 ## Release Notes

Makefile

@@ -15,6 +15,7 @@ CHARTS_INTEGRATION    := wire-server databases-ephemeral fake-aws
 # this list could be generated from the folder names under ./charts/ like so:
 # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts)
 CHARTS_RELEASE        := wire-server databases-ephemeral fake-aws aws-ingress backoffice calling-test demo-smtp elasticsearch-curator elasticsearch-external fluent-bit minio-external cassandra-external nginx-ingress-controller nginx-ingress-services reaper wire-server-metrics sftd
+BUILDAH_PUSH          ?= 1
 
 default: fast
 
@@ -218,6 +219,7 @@ libzauth:
 # Run this again after changes to libraries or dependencies.
 .PHONY: hie.yaml
 hie.yaml:
+	stack build implicit-hie
 	stack exec gen-hie > hie.yaml
 
 #####################################
@@ -242,16 +244,30 @@ hie.yaml:
 #   - kubectl
 #   - a valid kubectl context configured (i.e. access to a kubernetes cluster)
 .PHONY: kube-integration
-kube-integration: charts-integration
+kube-integration: guard-tag charts-integration
 	# by default "test-<your computer username> is used as namespace
 	# you can override the default by setting the NAMESPACE environment variable
 	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup.sh
 	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-test.sh
 
+.PHONY: kube-integration-setup
+kube-integration-setup: guard-tag charts-integration
+	# by default "test-<your computer username> is used as namespace
+	# you can override the default by setting the NAMESPACE environment variable
+	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup.sh
+
 .PHONY: kube-integration-teardown
 kube-integration-teardown:
 	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-teardown.sh
 
+.PHONY: kube-integration-setup-federation
+kube-integration-setup-federation: guard-tag charts-integration
+	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup-federation.sh
+
+.PHONY: kube-integration-federation
+kube-integration-federation:
+	cd services/brig && ./federation-tests.sh $(NAMESPACE)
+
 .PHONY: latest-brig-tag
 latest-brig-tag:
 	./hack/bin/find-latest-docker-tag.sh
@@ -268,6 +284,12 @@ release-chart-%:
 	fi
 	make chart-$(*)
 
+.PHONY: guard-tag
+guard-tag:
+	@if [ "${DOCKER_TAG}" = "${USER}" ]; then \
+	      echo "Environment variable DOCKER_TAG not set to non-default value. Re-run with DOCKER_TAG=<something>. Try using 'make latest-brig-tag' for latest develop docker image tag";\
+	    exit 1; \
+	fi
 
 # Rationale for copying charts to a gitignored folder before modifying helm versions and docker image tags:
 #
@@ -319,3 +341,17 @@ upload-charts: charts-release
 .PHONY: echo-release-charts
 echo-release-charts:
 	@echo ${CHARTS_RELEASE}
+
+.PHONY: buildah-docker
+buildah-docker:
+	./hack/bin/buildah-compile.sh
+	BUILDAH_PUSH=${BUILDAH_PUSH} ./hack/bin/buildah-make-images.sh
+
+.PHONY: buildah-docker-%
+buildah-docker-%:
+	./hack/bin/buildah-compile.sh $(*)
+	BUILDAH_PUSH=${BUILDAH_PUSH} EXECUTABLES=$(*) ./hack/bin/buildah-make-images.sh
+
+.PHONY: buildah-clean
+buildah-clean:
+	./hack/bin/buildah-clean.sh

build/alpine/Dockerfile.fast-intermediate

@@ -0,0 +1,26 @@
+# Produces intermediate docker image with all executables under /dist using fast option
+
+# Requires docker version >= 17.05 (requires support for multi-stage builds)
+# Requires to have created the wire-server-builder and wire-server-deps docker images (run `make` in this directory)
+# Usage example:
+#   (from wire-server root directory)
+#   docker build -f build/alpine/Dockerfile.fastintermediate .
+
+ARG builder=quay.io/wire/alpine-builder
+ARG deps=quay.io/wire/alpine-deps
+
+#--- Builder stage ---
+FROM ${builder} as builder
+
+WORKDIR /wire-server/
+
+COPY . /wire-server/
+
+RUN make clean fast
+
+#--- Minified stage ---
+FROM ${deps}
+
+COPY --from=builder /wire-server/dist/ /dist/
+# brig also needs some templates.
+COPY --from=builder /wire-server/services/brig/deb/opt/brig/templates/ /dist/templates/

charts/brig/templates/configmap.yaml

@@ -44,6 +44,15 @@ data:
       host: gundeck
       port: 8080
 
+    # TODO remove this
+    federator:
+      host: federator
+      port: 8080
+
+    federatorInternal:
+      host: federator
+      port: 8080
+
     {{- with .aws }}
     aws:
       prekeyTable: {{ .prekeyTable }}

charts/brig/templates/tests/brig-integration.yaml

@@ -53,7 +53,7 @@ spec:
     # same file-system.
     # The other test, "user.auth.cookies.limit", is skipped as it is flaky.
     # This is tracked in https://github.com/zinfra/backend-issues/issues/1150.
-    command: ["brig-integration", "--pattern", "!/turn/ && !/user.auth.cookies.limit/"]
+    command: ["brig-integration", "--pattern", "!/turn/ && !/user.auth.cookies.limit/ && !/brig-federation/"]
     volumeMounts:
     - name: "brig-integration"
       mountPath: "/etc/wire/integration"

charts/brig/templates/tests/configmap.yaml

@@ -26,6 +26,19 @@ data:
       host: spar
       port: 8080
 
+    # TODO remove this
+    federator:
+      host: federator
+      port: 8080
+
+    federatorInternal:
+      host: federator
+      port: 8080
+
+    federatorExternal:
+      host: federator
+      port: 8081
+
     nginz:
       # Full URL is set so that there can be a common cookiedomain between nginz and brig
       # needed by some integration tests
@@ -38,3 +51,21 @@ data:
       cert: /etc/wire/integration-secrets/provider-publiccert.pem
       botHost: https://brig-integration
       botPort: 9000
+
+    backendTwo:
+      brig:
+        host: brig.{{ .Release.Namespace }}-fed2.svc.cluster.local
+        port: 8080
+
+      # TODO remove this
+      federator:
+        host: federator.{{ .Release.Namespace }}-fed2.svc.cluster.local
+        port: 8080
+
+      federatorInternal:
+        host: federator.{{ .Release.Namespace }}-fed2.svc.cluster.local
+        port: 8080
+
+      federatorExternal:
+        host: federator.{{ .Release.Namespace }}-fed2.svc.cluster.local
+        port: 8081

charts/federator/templates/configmap.yaml

@@ -8,16 +8,32 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 data:
-  {{- with .Values.config }}
   federator.yaml: |
+
+    # TODO: delete this
+    federator:
+      host: 0.0.0.0
+      port: {{ .Values.service.internalFederatorPort }}
+
+    federatorInternal:
+      host: 0.0.0.0
+      port: {{ .Values.service.internalFederatorPort }}
+
+    federatorExternal:
+      host: 0.0.0.0
+      port: {{ .Values.service.externalFederatorPort }}
+
+    brig:
+      host: brig
+      port: 8080
+
+    {{- with .Values.config }}
+
     logNetStrings: True # log using netstrings encoding:
                         # http://cr.yp.to/proto/netstrings.txt
     logFormat: {{ .logFormat }}
     logLevel: {{ .logLevel }}
 
-    federator:
-      host: 0.0.0.0
-      port: 8080
 
     {{- with .optSettings }}
     optSettings:

charts/federator/templates/deployment.yaml

@@ -39,16 +39,20 @@ spec:
             - name: "federator-config"
               mountPath: "/etc/wire/federator/conf"
           ports:
-            - containerPort: {{ .Values.service.internalPort }}
-          livenessProbe:
-            httpGet:
-              scheme: HTTP
-              path: /i/status
-              port: {{ .Values.service.internalPort }}
-          readinessProbe:
-            httpGet:
-              scheme: HTTP
-              path: /i/status
-              port: {{ .Values.service.internalPort }}
+            - name: internal
+              containerPort: {{ .Values.service.internalFederatorPort }}
+            - name: external
+              containerPort: {{ .Values.service.externalFederatorPort }}
+          # TODO ensure to have a status endpoint!
+          # livenessProbe:
+          #   httpGet:
+          #     scheme: HTTP
+          #     path: /i/status
+          #     port: {{ .Values.service.internalFederatorPort }}
+          # readinessProbe:
+          #   httpGet:
+          #     scheme: HTTP
+          #     path: /i/status
+          #     port: {{ .Values.service.internalFederatorPort }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}

charts/federator/templates/service.yaml

@@ -11,8 +11,12 @@ spec:
   type: ClusterIP
   ports:
     - name: http
-      port: {{ .Values.service.externalPort }}
-      targetPort: {{ .Values.service.internalPort }}
+      port: {{ .Values.service.internalFederatorPort }}
+      targetPort: {{ .Values.service.internalFederatorPort }}
+
+    - name: external
+      port: {{ .Values.service.externalFederatorPort }}
+      targetPort: {{ .Values.service.externalFederatorPort }}
   selector:
     wireService: federator
     release: {{ .Release.Name }}

charts/federator/values.yaml

@@ -5,8 +5,8 @@ image:
   tag: do-not-use
 
 service:
-  externalPort: 8080
-  internalPort: 8080
+  internalFederatorPort: 8080
+  externalFederatorPort: 8081
 
 resources:
   # FUTUREWORK: come up with numbers which didn't appear out of thin air

charts/nginz/values.yaml

@@ -190,6 +190,11 @@ nginx_conf:
       - staging
       disable_zauth: true
       basic_auth: true
+    - path: ~* ^/i/users/([^/]*)/rich-info
+      envs:
+      - staging
+      disable_zauth: true
+      basic_auth: true
     - path: ~* ^/i/teams/([^/]*)/suspend
       envs:
       - staging
@@ -200,6 +205,11 @@ nginx_conf:
       - staging
       disable_zauth: true
       basic_auth: true
+    - path: /i/provider/activation-code
+      envs:
+      - staging
+      disable_zauth: true
+      basic_auth: true
     - path: /cookies
       envs:
       - all
@@ -245,6 +255,12 @@ nginx_conf:
     - path: /calls
       envs:
       - all
+    - path: ~* ^/teams/([^/]*)/size$
+      envs:
+      - all
+    - path: ~* ^/teams/([^/]*)/search$
+      envs:
+      - all
     galley:
     - path: /conversations/code-check
       disable_zauth: true
@@ -287,6 +303,9 @@ nginx_conf:
     - path: ~* ^/teams/([^/]*)/conversations(.*)
       envs:
       - all
+    - path: ~* ^/teams/([^/]*)/members/csv$
+      envs:
+      - all
     - path: ~* ^/teams/([^/]*)/legalhold(.*)
       envs:
       - all
@@ -314,6 +333,11 @@ nginx_conf:
     - path: ~* ^/teams/([^/]*)/features/([^/])*
       envs:
       - all
+    - path: ~* /i/teams/([^/]*)/features/([^/]*)
+      envs:
+      - staging
+      disable_zauth: true
+      basic_auth: true
     gundeck:
     - path: /push
       envs:

charts/spar/templates/tests/configmap.yaml

@@ -15,3 +15,6 @@ data:
     spar:
       host: spar
       port: 8080
+
+    # Keep this in sync with brigs setTeamInvitationTimeout
+    brigSettingsTeamInvitationTimeout: 10