Merge pull request #1092 from wireapp/release_2020_05_07

.headroom.yaml

@@ -0,0 +1,31 @@
+run-mode: replace
+source-paths:
+- libs
+- services
+- tools
+excluded-paths:
+- '\.stack-work'
+template-paths:
+- tools/headroom-templates
+variables:
+  organization: Wire Swiss GmbH
+  email: opensource@wire.com
+  project: This file is part of the Wire Server implementation.
+  year: '2020'
+license-headers:
+  haskell:
+    file-extensions: ["hs", "hsc"]
+    # after pragmas, to make Ormolu happy
+    put-after: ["^{-#"]
+    # but before the module header (or comment) in any case
+    put-before: ["^module", "^-- \\|"]
+    margin-after: 1
+    margin-before: 1
+    line-comment:
+      prefixed-by: "--"
+  rust:
+    file-extensions: ["rs"]
+    margin-after: 1
+    margin-before: 1
+    line-comment:
+      prefixed-by: "//"

CHANGELOG.md

@@ -1,3 +1,45 @@
+# 2020-05-07
+
+## Upgrade steps (IMPORTANT)
+
+* Deploy new version of all services as usual, make sure `enableIndexedBillingTeamMember` setting in galley is `false`.
+* Run backfill using
+  ```bash
+  CASSANDRA_HOST_GALLEY=<IP Address of one of the galley cassandra instaces>
+  CASSANDRA_PORT_GALLEY=<port>
+  CASSANDRA_KEYSPACE_GALLEY=<GALLEY_KEYSPACE>
+  docker run quay.io/wire/backfill-billing-team-members:2.81.18 \
+    --cassandra-host-galley="$CASSANDRA_HOST_GALLEY" \
+    --cassandra-port-galley="$CASSANDRA_PORT_GALLEY" \
+    --cassandra-keyspace-galley="$CASSANDRA_KEYSPACE_GALLEY"
+  ```
+  You can also run the above using [`kubectl run`](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#run).
+* Set `enableIndexedBillingTeamMember` setting in galley to `true` and re-deploy the same version.
+
+## New Features
+
+* Custom search visibility - limit name search (#1086)
+* Add tool to backfill billing_team_member (#1089)
+* Index billing team members (#1081, #1091)
+* Allow team deletion on stern (#1080)
+* Do not fanout very large teams (#1060, #1075)
+
+## Bug fixes
+
+* Fix licenses of db tools (#1088)
+
+## Internal Changes
+* Add docs for updating ID Provider (#1074)
+* Add comments/docs about hie.yaml (#1037)
+* Don't poll from SQS as often (#1082)
+* Refactor: Split API modules into public/internal (#1083)
+* Manage license headers with headroom instead of licensure (#1084)
+* Monitor access to DynamoDB (#1077)
+* Make make docker-intermediate command work again (#1079)
+* Upgrade Ormolu to 0.0.5.0 (#1078)
+* Add (very few) unit tests to galley (#1071)
+* Pull brig-index before running the docker ephemeral setup (#1066)
+
 # 2020-04-21
 
 ## New Features

Makefile

@@ -48,16 +48,15 @@ formatf:
 formatc:
 	./tools/ormolu.sh -c
 
-# For any Haskell or Rust file that doesn't mention AGPL yet, add a license header.
-# It's your own reponsibility to keep ormolu happy.
+# For any Haskell or Rust file, update or add a license header if necessary.
+# Headers should be added according to Ormolu's formatting rules, but please check just in case.
 .PHONY: add-license
 add-license:
-	for file in $$(git grep -L "GNU Affero General Public License" | grep '\.hs$$\|\.hsc$$\|\.rs$$'); do \
-		echo "Adding license to $${file}."; \
-		licensure -i $${file}; \
-	done;
+	# Check headroom is installed. If not, please run 'stack install headroom'
+	command -v headroom
+	headroom run
 	@echo ""
-	@echo "you most probably want to run 'make formatf' now to keep ormolu happy"
+	@echo "you might want to run 'make formatf' now to make sure ormolu is happy"
 
 # Clean
 .PHONY: clean
@@ -137,7 +136,7 @@ docker-builder:
 .PHONY: docker-intermediate
 docker-intermediate:
 	# `docker-intermediate` needs to be built whenever code changes - this essentially runs `stack clean && stack install` on the whole repo
-	docker build -t $(DOCKER_USER)/alpine-intermediate:$(DOCKER_TAG) -f build/alpine/Dockerfile.intermediate --build-arg builder=$(DOCKER_USER)/alpine-builder --build-arg deps=$(DOCKER_USER)/alpine-deps .;
+	docker build -t $(DOCKER_USER)/alpine-intermediate:$(DOCKER_TAG) -f build/alpine/Dockerfile.intermediate --build-arg builder=$(DOCKER_USER)/alpine-builder:develop --build-arg deps=$(DOCKER_USER)/alpine-deps:develop .;
 	docker tag $(DOCKER_USER)/alpine-intermediate:$(DOCKER_TAG) $(DOCKER_USER)/alpine-intermediate:latest;
 	if test -n "$$DOCKER_PUSH"; then docker login -u $(DOCKER_USERNAME) -p $(DOCKER_PASSWORD); docker push $(DOCKER_USER)/alpine-intermediate:$(DOCKER_TAG); docker push $(DOCKER_USER)/alpine-intermediate:latest; fi;
 
@@ -193,6 +192,11 @@ db-reset:
 libzauth:
 	$(MAKE) -C libs/libzauth install
 
+# Useful when using Haskell IDE Engine
+# https://github.com/haskell/haskell-ide-engine
+# requires 'yq' executable on your path.
+#
+# Run this again after changes to libraries or dependencies.
 .PHONY: hie.yaml
 hie.yaml:
 	./tools/gen-hie-yaml.sh > hie.yaml

deploy/dockerephemeral/run.sh

@@ -15,8 +15,6 @@ docker pull quay.io/wire/brig-schema
 docker pull quay.io/wire/galley-schema
 docker pull quay.io/wire/gundeck-schema
 docker pull quay.io/wire/spar-schema
-
-# elasticsearch does not do migrations, so the following line is not needed.
-#docker pull quay.io/wire/brig-index
+docker pull quay.io/wire/brig-index
 
 docker-compose --file "$DOCKER_FILE" up

docs/developer/dependencies.md

@@ -98,20 +98,11 @@ source $HOME/.cargo/env
 
 ## Formatting Haskell files
 
-You need `ormolu` on your PATH, get it with `stack install ormolu`
+You need [`ormolu`](https://github.com/tweag/ormolu) on your PATH, get it with `stack install ormolu`
 
 ## Generating license headers
 
-We use [`licensure`](https://github.com/chasinglogic/licensure).
-
-To get it:
-
-```
-git clone https://github.com/chasinglogic/licensure
-cd licensure
-git checkout 96d7fad9201e19e5193bf7435ab5fdf4a6ad4685  # master as of 2020-02-18
-cargo install --path .
-```
+We use [`headroom`](https://github.com/vaclavsvejcar/headroom), get it with `stack install headroom`
 
 ## makedeb
 

docs/developer/editor-setup.md

@@ -2,6 +2,18 @@
 
 This page provides tips for setting up editors to work with the Wire codebase.
 
+## For multiple editors {#DevAll}
+
+### Using Haskell IDE Engine
+
+See [official documentation](https://github.com/haskell/haskell-ide-engine)
+
+In addition, you can generate (and re-generate after changes to stack.yaml) a `hie.yaml` configuration file with
+
+```
+make hie.yaml
+```
+
 ## Emacs {#DevEmacs}
 
 ### Jump-to-definition {#DevEmacsJump}
@@ -55,7 +67,10 @@ here](https://github.com/tweag/ormolu#editor-integration).
 
 ## Vim {#DevVim}
 
-TODO.
+### hie (Haskell IDE engine) integration
+
+* Generate `hie.yaml` as described [at the top of this file](#using-haskell-ide-engine)
+* You may follow the setup described [in this blog post](http://marco-lopes.com/articles/Vim-and-Haskell-in-2019/)
 
 ### ormolu integration
 

docs/reference/cassandra-schema.cql

@@ -279,6 +279,26 @@ CREATE TABLE galley_test.legalhold_service (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+CREATE TABLE galley_test.billing_team_member (
+    team uuid,
+    user uuid,
+    PRIMARY KEY (team, user)
+) WITH CLUSTERING ORDER BY (user ASC)
+    AND bloom_filter_fp_chance = 0.01
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
 CREATE TABLE galley_test.conversation_codes (
     key ascii,
     scope int,
@@ -912,6 +932,8 @@ CREATE TABLE brig_test.team_invitation (
     created_at timestamp,
     created_by uuid,
     email text,
+    name text,
+    phone text,
     role int,
     PRIMARY KEY (team, id)
 ) WITH CLUSTERING ORDER BY (id ASC)
@@ -1362,7 +1384,9 @@ CREATE TABLE spar_test.idp (
     idp uuid PRIMARY KEY,
     extra_public_keys list<blob>,
     issuer text,
+    old_issuers list<text>,
     public_key blob,
+    replaced_by uuid,
     request_uri text,
     team uuid
 ) WITH bloom_filter_fp_chance = 0.1

docs/reference/config-options.md

@@ -5,6 +5,26 @@ Fragment.
 This page is about the yaml files that determine the configuration of
 the Wire backend services.
 
+## Settings
+
+```
+# [galley.yaml]
+settings:
+  enableIndexedBillingTeamMembers: false
+```
+
+### Indexed Billing Team Members
+
+Use indexed billing team members for journaling. When `enabled`,
+galley would use the `billing_team_member` table to send billing
+events with user ids of team owners (who have the `SetBilling`
+permission). Before enabling this flag, the `billing_team_member`
+table must be backfilled.
+
+Even when the flag is `disabled`, galley will keep writing to the
+`biling_team_member` table, this flag only affects the reads and has
+been added in order to deploy new code and backfill data in 
+production.
 
 ## Feature flags
 
@@ -17,6 +37,7 @@ settings:
   featureFlags:
     sso: disabled-by-default
     legalhold: disabled-by-default
+    teamSearchVisibility: disabled-by-default
 ```
 
 The `featureFlags` field in the galley settings is mandatory, and all
@@ -48,3 +69,13 @@ IMPORTANT: If you switch this back to `disabled-permanently` from
 `disabled-by-default`, LegalHold devices may still be active in teams
 that have created them while it was allowed.  This may change in the
 future.
+
+### Team Search Visibility
+
+Is a team allowed to change its team search visibility settings? If enabled
+for the team, it can be configured so that non team users do not show up in search.
+
+This sets the default setting for all teams, and can be overridden for
+individual teams by customer support / backoffice. [Allowed
+values](https://github.com/wireapp/wire-server/blob/custom-search-visibility-limit-name-search/libs/galley-types/src/Galley/Types/Teams.hs#L382-L385):
+`disabled-by-default`, `enabled-by-default`.