Release 2020/07/13

CHANGELOG.md

@@ -1,3 +1,35 @@
+# 2020-07-13
+
+## Release Notes
+
+* If you are self-hosting wire on the public internet, consider [changing your brig server config](https://github.com/wireapp/wire-server/blob/49f414add470f4c5e969814a37bc851e26f6d9a7/docs/reference/user/registration.md#blocking-creation-of-personal-users-new-teams-refrestrictregistration).
+* Deploy all services except nginz.
+* No migrations, no restrictions on deployment order.
+
+## New Features
+
+* Restrict user creation in on-prem installations (#1161)
+* Implement active flag in SCIM for user suspension (#1158)
+
+## Bug Fixes
+
+* Fix setting team feature status in Stern/backoffice (#1146)
+* Add missing Swagger models (#1153)
+* docs/reference/elastic-search.md: fix typos (#1154)
+
+## Internal changes
+
+* Federation: Implement ID mapping (galley) (#1134)
+* Tweak cassandra container settings to get it to work on nixos. (#1155)
+* Merge wireapp/subtree-hscim repository under `/libs`, preserving history (#1152)
+* Add link to twilio message ID format (#1150)
+* Run backoffice locally (#1148)
+* Fix services-demo (#1149, #1156)
+* Add missing license headers (#1143)
+* Test sign up with invalid email (#1141)
+* Fix ormolu script (source code pretty-printing) (#1142)
+
+
 # 2020-06-19
 
 ## Release Notes

Makefile

@@ -161,6 +161,7 @@ docker-services:
 	$(MAKE) -C services/cannon docker
 	$(MAKE) -C services/proxy docker
 	$(MAKE) -C services/spar docker
+	$(MAKE) -C tools/stern docker
 	$(MAKE) docker-exe-zauth
 	$(MAKE) -C services/nginz docker
 

deploy/dockerephemeral/docker-compose.yaml

@@ -97,10 +97,18 @@ services:
     image: julialongtin/cassandra:0.0.9
     ports:
       - "127.0.0.1:9042:9042"
+    ulimits:
+      memlock: 65536
+      nofile: 100000
+      nproc: 32768
     environment:
-# what's present in the jvm.options file by default.
-#      - "CS_JAVA_OPTIONS=-Xmx1024M -Xms1024M -Xmn200M"
+      # what's present in the jvm.options file by default:
+      #- "CS_JAVA_OPTIONS=-Xmx1024M -Xms1024M -Xmn200M"
       - "CS_JVM_OPTIONS=-Xmx128M -Xms128M -Xmn50M"
+
+      # on nixos, you also may need to run
+      #   sysctl -w vm.max_map_count=1048576
+      # or add that to your `configuration.nix`
     networks:
       - demo_wire
 

deploy/services-demo/conf/brig.demo-docker.yaml

@@ -84,6 +84,9 @@ zauth:
     sessionTokenTimeout: 604800  # 7 days
     accessTokenTimeout: 900      # 15 minutes
     providerTokenTimeout: 604800 # 7 days
+    legalHoldUserTokenTimeout: 4838400    # 56 days
+    legalHoldSessionTokenTimeout: 604800  # 7 days
+    legalHoldAccessTokenTimeout: 900 # 15 minutes
 
 turn:
   serversV2: resources/turn/servers-v2.txt

deploy/services-demo/conf/galley.demo-docker.yaml

@@ -26,6 +26,13 @@ settings:
   maxConvSize: 128
   intraListing: false
   conversationCodeURI: https://cannon/join/
+  concurrentDeletionEvents: 1024
+  deleteConvThrottleMillis: 0
+
+  featureFlags:  # see #RefConfigOptions in `/docs/reference`
+    sso: disabled-by-default
+    legalhold: disabled-by-default
+    teamSearchVisibility: disabled-by-default
 
 logLevel: Info
 logNetStrings: false

deploy/services-demo/conf/galley.demo.yaml

@@ -32,6 +32,7 @@ settings:
   featureFlags:  # see #RefConfigOptions in `/docs/reference`
     sso: disabled-by-default
     legalhold: disabled-by-default
+    teamSearchVisibility: disabled-by-default
 
 logLevel: Info
 logNetStrings: false

deploy/services-demo/conf/nginz/nginx-docker.conf

@@ -58,8 +58,12 @@ http {
   #
   # Logging
   #
+  # Note sanitized_request:
+  # We allow passing access_token as query parameter for e.g. websockets
+  # However we do not want to log access tokens.
+  #
 
-  log_format custom_zeta '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" - $connection $request_time $upstream_response_time $upstream_cache_status $zauth_user $zauth_connection $request_id $proxy_protocol_addr';
+  log_format custom_zeta '$remote_addr - $remote_user [$time_local] "$sanitized_request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" - $connection $request_time $upstream_response_time $upstream_cache_status $zauth_user $zauth_connection $request_id $proxy_protocol_addr';
   access_log /dev/stdout custom_zeta;
 
   #
@@ -97,6 +101,8 @@ http {
   }
 
 
+  # Docker DNS, required to resolve the references to stern here.
+  resolver 127.0.0.11;
 
   #
   # Locations
@@ -192,6 +198,7 @@ http {
       include common_response_with_zauth.conf;
       proxy_pass http://brig;
     }
+
     # Cargohold Endpoints
 
     rewrite ^/api-docs/assets  /assets/api-docs?base_url=http://127.0.0.1:8080/ break;
@@ -272,6 +279,11 @@ http {
       proxy_pass http://galley;
     }
 
+    location ~* ^/teams/([^/]*)/features/([^/]*) {
+      include common_response_with_zauth.conf;
+      proxy_pass http://galley;
+    }
+
     # Gundeck Endpoints
 
     rewrite ^/api-docs/push  /push/api-docs?base_url=http://127.0.0.1:8080/ break;
@@ -345,6 +357,28 @@ http {
       proxy_pass http://spar;
     }
 
+    # Stern Endpoints
+
+    # We add a `/stern` suffix to the URL to resolve clashes with non-Stern endpoints.
+    rewrite ^/backoffice/api-docs/stern  /stern/api-docs?base_url=http://127.0.0.1:8080/stern/ break;
+
+    location /stern/api-docs {
+      include common_response_no_zauth.conf;
+      # Using a variable instead of plain upstream makes nginx still start up if stern is not there.
+      # https://sandro-keil.de/blog/let-nginx-start-if-upstream-host-is-unavailable-or-down
+      set $stern stern:8091;
+      proxy_pass http://$stern;
+    }
+
+    location /stern {
+      include common_response_no_zauth.conf;
+      # Using a variable instead of plain upstream makes nginx still start up if stern is not there.
+      # https://sandro-keil.de/blog/let-nginx-start-if-upstream-host-is-unavailable-or-down
+      set $stern stern:8091;
+      # The trailing slash matters, as it makes sure the `/stern` prefix is removed.
+      proxy_pass http://$stern/;
+    }
+
     #
     # Swagger Resource Listing
     #
@@ -363,6 +397,24 @@ http {
         more_set_headers 'Access-Control-Allow-Origin: $http_origin';
     }
 
+    #
+    # Back Office Swagger Resource Listing
+    #
+    location /backoffice/api-docs {
+        zauth off;
+        default_type application/json;
+        root conf/nginz/zwagger-ui;
+        index resources.json;
+        if ($request_method = 'OPTIONS') {
+              add_header 'Access-Control-Allow-Methods' "GET, POST, PUT, DELETE, OPTIONS";
+              add_header 'Access-Control-Allow-Headers' "$http_access_control_request_headers, DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type";
+              add_header 'Content-Type' 'text/plain; charset=UTF-8';
+              add_header 'Content-Length' 0;
+              return 204;
+        }
+        more_set_headers 'Access-Control-Allow-Origin: $http_origin';
+    }
+
     # Swagger UI
 
     location /swagger-ui {

deploy/services-demo/conf/nginz/nginx.conf

@@ -354,6 +354,24 @@ http {
       proxy_pass http://spar;
     }
 
+    # Stern Endpoints
+
+    # We add a `/stern` suffix to the URL to resolve clashes with non-Stern endpoints.
+    rewrite ^/backoffice/api-docs/stern  /stern/api-docs?base_url=http://127.0.0.1:8080/stern/ break;
+
+    location /stern/api-docs {
+      include common_response_no_zauth.conf;
+      # We don't use an `upstream` for stern, since running stern is optional.
+      proxy_pass http://127.0.0.1:8091;
+    }
+
+    location /stern {
+      include common_response_no_zauth.conf;
+      # We don't use an `upstream` for stern, since running stern is optional.
+      # The trailing slash matters, as it makes sure the `/stern` prefix is removed.
+      proxy_pass http://127.0.0.1:8091/;
+    }
+
     #
     # Swagger Resource Listing
     #
@@ -372,6 +390,24 @@ http {
         more_set_headers 'Access-Control-Allow-Origin: $http_origin';
     }
 
+    #
+    # Back Office Swagger Resource Listing
+    #
+    location /backoffice/api-docs {
+        zauth off;
+        default_type application/json;
+        root conf/nginz/zwagger-ui;
+        index resources.json;
+        if ($request_method = 'OPTIONS') {
+              add_header 'Access-Control-Allow-Methods' "GET, POST, PUT, DELETE, OPTIONS";
+              add_header 'Access-Control-Allow-Headers' "$http_access_control_request_headers, DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type";
+              add_header 'Content-Type' 'text/plain; charset=UTF-8';
+              add_header 'Content-Length' 0;
+              return 204;
+        }
+        more_set_headers 'Access-Control-Allow-Origin: $http_origin';
+    }
+
     # Swagger UI
 
     location /swagger-ui {

deploy/services-demo/conf/proxy.demo-docker.yaml

@@ -0,0 +1,9 @@
+host: proxy
+port: 8087
+
+httpPoolSize: 1000
+maxConns: 5000
+secretsConfig: resources/proxy.config
+
+logLevel: Info
+logNetStrings: false

deploy/services-demo/conf/stern.demo-docker.yaml

@@ -0,0 +1,28 @@
+stern:
+  host: stern
+  port: 8091
+
+brig:
+  host: brig
+  port: 8082
+
+galley:
+  host: galley
+  port: 8085
+
+gundeck:
+  host: gundeck
+  port: 8086
+
+# Both ibis and galeb should be made optional for
+# installations where these services are not available
+galeb:
+  host: galeb
+  port: 8089
+
+ibis:
+  host: ibis
+  port: 8090
+
+logLevel: Info
+logNetStrings: false

deploy/services-demo/conf/stern.demo.yaml

@@ -0,0 +1,28 @@
+stern:
+  host: 127.0.0.1
+  port: 8091
+
+brig:
+  host: 127.0.0.1
+  port: 8082
+
+galley:
+  host: 127.0.0.1
+  port: 8085
+
+gundeck:
+  host: 127.0.0.1
+  port: 8086
+
+# Both ibis and galeb should be made optional for
+# installations where these services are not available
+galeb:
+  host: 127.0.0.1
+  port: 8089
+
+ibis:
+  host: 127.0.0.1
+  port: 8090
+
+logLevel: Info
+logNetStrings: false

deploy/services-demo/demo.sh

@@ -4,15 +4,19 @@
 
 set -eo pipefail
 
-USAGE="$0 [docker]"
-MODE="$1"
+USAGE="$0 [docker] [--run-backoffice]"
 docker_deployment="false"
-if [ "$MODE" = "docker" ]; then
+if [ "$1" = "docker" ] || [ "$2" = "docker" ] ; then
     docker_deployment="true"
 fi
+run_backoffice="false"
+if [ "$1" = "--run-backoffice" ] || [ "$2" = "--run-backoffice" ] ; then
+    run_backoffice="true"
+fi
 TOP_LEVEL="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 DOCKER_FILE="$SCRIPT_DIR/docker-compose.yaml"
+DOCKER_FILE_BACKOFFICE="$SCRIPT_DIR/docker-compose-backoffice.yaml"
 DIR="${TOP_LEVEL}/services"
 PARENT_PID=$$
 rm -f /tmp/demo.* # remove previous temp files, if any
@@ -32,7 +36,7 @@ function list_descendants () {
 }
 
 function kill_gracefully() {
-    pkill "gundeck|brig|galley|cargohold|cannon|spar"
+    pkill "gundeck|brig|galley|cargohold|cannon|spar|stern"
     sleep 1
     kill $(list_descendants $PARENT_PID) &> /dev/null
 }
@@ -83,7 +87,8 @@ function check_prerequisites() {
             && test -f ${DIR}/../dist/cargohold \
             && test -f ${DIR}/../dist/proxy \
             && test -f ${DIR}/../dist/spar \
-            && test -f ${DIR}/../dist/nginx \
+            && test -f ${DIR}/../dist/stern \
+            && ( test -f ${DIR}/../dist/nginx || which nix-build ) \
             || { echo "Not all services are compiled. How about you run 'cd ${TOP_LEVEL} && make services' first?"; exit 1; }
     fi
 }
@@ -107,8 +112,18 @@ function run_haskell_service() {
 function run_nginz() {
     colour=$1
     prefix=$([ -w /usr/local ] && echo /usr/local || echo "${HOME}/.wire-dev")
-    (cd ${SCRIPT_DIR} && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${prefix}/lib/ ${DIR}/../dist/nginx -p ${SCRIPT_DIR} -c ${SCRIPT_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \
-        | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" &
+
+    # For nix we dont need LD_LIBRARY_PATH; we link against libzauth directly.
+    # nix-build will put a symlink to ./result with the nginx artifact
+    if which nix-build; then
+      nginz=$(nix-build "${DIR}/../nix" -A nginz --no-out-link )
+      (cd ${SCRIPT_DIR} && ${nginz}/bin/nginx -p ${SCRIPT_DIR} -c ${SCRIPT_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \
+          | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" &
+    else
+      prefix=$([ -w /usr/local ] && echo /usr/local || echo "${HOME}/.wire-dev")
+      (cd ${SCRIPT_DIR} && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${prefix}/lib/ ${DIR}/../dist/nginx -p ${SCRIPT_DIR} -c ${SCRIPT_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \
+          | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" &
+    fi
 }
 
 function copy_brig_templates() {
@@ -146,9 +161,16 @@ if [ "$docker_deployment" = "false" ]; then
     run_haskell_service cargohold ${purpleish}
     run_haskell_service proxy ${redish}
     run_haskell_service spar ${orange}
+    if [ "$run_backoffice" = "true" ]; then
+        run_haskell_service stern ${orange}
+    fi
     run_nginz ${blueish}
 else
-    docker-compose --file "$DOCKER_FILE" up
+    if [ "$run_backoffice" = "true" ]; then
+        docker-compose --file "$DOCKER_FILE" --file "$DOCKER_FILE_BACKOFFICE" up
+    else
+        docker-compose --file "$DOCKER_FILE" up
+    fi
 fi
 
 sleep 3 # wait a moment for services to start before continuing