Brig integration tests

charts/brig/templates/configmap.yaml

@@ -44,6 +44,15 @@ data:
       host: gundeck
       port: 8080
 
+    # TODO remove this
+    federator:
+      host: federator
+      port: 8080
+
+    federatorInternal:
+      host: federator
+      port: 8080
+
     {{- with .aws }}
     aws:
       prekeyTable: {{ .prekeyTable }}

charts/brig/templates/tests/configmap.yaml

@@ -26,6 +26,19 @@ data:
       host: spar
       port: 8080
 
+    # TODO remove this
+    federator:
+      host: federator
+      port: 8080
+
+    federatorInternal:
+      host: federator
+      port: 8080
+
+    federatorExternal:
+      host: federator
+      port: 8081
+
     nginz:
       # Full URL is set so that there can be a common cookiedomain between nginz and brig
       # needed by some integration tests
@@ -38,3 +51,21 @@ data:
       cert: /etc/wire/integration-secrets/provider-publiccert.pem
       botHost: https://brig-integration
       botPort: 9000
+
+    backendTwo:
+      brig:
+        host: brig.{{ .Release.Namespace }}-fed2.svc.cluster.local
+        port: 8080
+
+      # TODO remove this
+      federator:
+        host: federator.{{ .Release.Namespace }}-fed2.svc.cluster.local
+        port: 8080
+
+      federatorInternal:
+        host: federator.{{ .Release.Namespace }}-fed2.svc.cluster.local
+        port: 8080
+
+      federatorExternal:
+        host: federator.{{ .Release.Namespace }}-fed2.svc.cluster.local
+        port: 8081

charts/federator/templates/configmap.yaml

@@ -8,16 +8,32 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 data:
-  {{- with .Values.config }}
   federator.yaml: |
+
+    # TODO: delete this
+    federator:
+      host: 0.0.0.0
+      port: {{ .Values.service.internalFederatorPort }}
+
+    federatorInternal:
+      host: 0.0.0.0
+      port: {{ .Values.service.internalFederatorPort }}
+
+    federatorExternal:
+      host: 0.0.0.0
+      port: {{ .Values.service.externalFederatorPort }}
+
+    brig:
+      host: brig
+      port: 8080
+
+    {{- with .Values.config }}
+
     logNetStrings: True # log using netstrings encoding:
                         # http://cr.yp.to/proto/netstrings.txt
     logFormat: {{ .logFormat }}
     logLevel: {{ .logLevel }}
 
-    federator:
-      host: 0.0.0.0
-      port: 8080
 
     {{- with .optSettings }}
     optSettings:

charts/federator/templates/deployment.yaml

@@ -39,16 +39,20 @@ spec:
             - name: "federator-config"
               mountPath: "/etc/wire/federator/conf"
           ports:
-            - containerPort: {{ .Values.service.internalPort }}
-          livenessProbe:
-            httpGet:
-              scheme: HTTP
-              path: /i/status
-              port: {{ .Values.service.internalPort }}
-          readinessProbe:
-            httpGet:
-              scheme: HTTP
-              path: /i/status
-              port: {{ .Values.service.internalPort }}
+            - name: internal
+              containerPort: {{ .Values.service.internalFederatorPort }}
+            - name: external
+              containerPort: {{ .Values.service.externalFederatorPort }}
+          # TODO ensure to have a status endpoint!
+          # livenessProbe:
+          #   httpGet:
+          #     scheme: HTTP
+          #     path: /i/status
+          #     port: {{ .Values.service.internalFederatorPort }}
+          # readinessProbe:
+          #   httpGet:
+          #     scheme: HTTP
+          #     path: /i/status
+          #     port: {{ .Values.service.internalFederatorPort }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}

charts/federator/templates/service.yaml

@@ -11,8 +11,12 @@ spec:
   type: ClusterIP
   ports:
     - name: http
-      port: {{ .Values.service.externalPort }}
-      targetPort: {{ .Values.service.internalPort }}
+      port: {{ .Values.service.internalFederatorPort }}
+      targetPort: {{ .Values.service.internalFederatorPort }}
+
+    - name: external
+      port: {{ .Values.service.externalFederatorPort }}
+      targetPort: {{ .Values.service.externalFederatorPort }}
   selector:
     wireService: federator
     release: {{ .Release.Name }}

charts/federator/values.yaml

@@ -5,8 +5,8 @@ image:
   tag: do-not-use
 
 service:
-  externalPort: 8080
-  internalPort: 8080
+  internalFederatorPort: 8080
+  externalFederatorPort: 8081
 
 resources:
   # FUTUREWORK: come up with numbers which didn't appear out of thin air

hack/bin/integration-setup.sh

@@ -24,10 +24,18 @@ done
 echo "Installing charts..."
 
 function printLogs() {
+    echo "---- a command failed, attempting to print useful debug information..."
+    echo "-------------------------------"
+    echo "-------------------------------"
+    echo "-------------------------------"
+    echo ""
+    kubectl -n ${NAMESPACE} get pods
     kubectl -n ${NAMESPACE} get pods | grep -v Running | grep -v Pending | grep -v Completed | grep -v STATUS | grep -v ContainerCreating | awk '{print $1}' | xargs -n 1 -I{} bash -c "printf '\n\n----LOGS FROM {}:\n'; kubectl -n ${NAMESPACE} logs --tail=30 {}" || true
     kubectl -n ${NAMESPACE} get pods | grep Pending | awk '{print $1}' | xargs -n 1 -I{} bash -c "printf '\n\n----DESCRIBE 'pending' {}:\n'; kubectl -n ${NAMESPACE} describe pod {}" || true
 }
 
+trap printLogs ERR
+
 FEDERATION_DOMAIN="$NAMESPACE.svc.cluster.local"
 
 for chart in "${charts[@]}"; do

services/brig/brig.cabal

@@ -4,7 +4,7 @@ cabal-version: 1.12
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: c5312f061d226f10f6e7e3dd06dce7730af75b112fd4251c5c7455dc1ab25a4f
+-- hash: 83620e2ed26c5349fbca70be3e5e3978d85924a7bfbcdb9db63eac26cf8693c4
 
 name:           brig
 version:        1.35.0
@@ -294,6 +294,7 @@ executable brig-integration
       API.User.RichInfo
       API.User.Util
       API.UserPendingActivation
+      Federation.User
       Index.Create
       Util
       Util.AWS

services/brig/brig.integration.yaml

@@ -24,9 +24,9 @@ gundeck:
   host: 127.0.0.1
   port: 8086
 
-# federator:
-#   host: 127.0.0.1
-#   port: 8097
+federatorInternal:
+  host: 127.0.0.1
+  port: 8097
 
 # You can set up local SQS/Dynamo running e.g. `../../deploy/dockerephemeral/run.sh`
 aws:

services/brig/src/Brig/App.hs

@@ -32,6 +32,7 @@ module Brig.App
     cargohold,
     galley,
     gundeck,
+    federator,
     userTemplates,
     providerTemplates,
     teamTemplates,
@@ -141,6 +142,7 @@ data Env = Env
   { _cargohold :: RPC.Request,
     _galley :: RPC.Request,
     _gundeck :: RPC.Request,
+    _federator :: Maybe Endpoint, -- FUTUREWORK: should we use a better type here? E.g. to avoid fresh connections all the time?
     _casClient :: Cas.ClientState,
     _smtpEnv :: Maybe SMTP.SMTP,
     _emailSender :: Email,
@@ -215,6 +217,7 @@ newEnv o = do
       { _cargohold = mkEndpoint $ Opt.cargohold o,
         _galley = mkEndpoint $ Opt.galley o,
         _gundeck = mkEndpoint $ Opt.gundeck o,
+        _federator = Opt.federatorInternal o,
         _casClient = cas,
         _smtpEnv = emailSMTP,
         _emailSender = Opt.emailSender . Opt.general . Opt.emailSMS $ o,

services/brig/src/Brig/Options.hs

@@ -344,7 +344,7 @@ data Opts = Opts
     -- | Gundeck address
     gundeck :: !Endpoint,
     -- | Federator address
-    federator :: !(Maybe Endpoint),
+    federatorInternal :: !(Maybe Endpoint),
     -- external
 
     -- | Cassandra settings

services/brig/test/integration/API/Provider.hs

@@ -85,9 +85,8 @@ import Test.Tasty.HUnit
 import Util
 import Web.Cookie (SetCookie (..), parseSetCookie)
 
-tests :: Maybe Config -> Manager -> DB.ClientState -> Brig -> Cannon -> Galley -> IO TestTree
-tests mbConf p db b c g = do
-  conf <- maybe getEnvConfig pure mbConf
+tests :: Config -> Manager -> DB.ClientState -> Brig -> Cannon -> Galley -> IO TestTree
+tests conf p db b c g = do
   return $
     testGroup
       "provider"
@@ -157,16 +156,6 @@ data Config = Config
 
 instance FromJSON Config
 
--- | Get the config from environment variables (and some defaults)
-getEnvConfig :: IO Config
-getEnvConfig = do
-  privateKey <- getEnv "TEST_KEY"
-  publicKey <- getEnv "TEST_PUBKEY"
-  cert <- getEnv "TEST_CERT"
-  let botHost = "https://localhost"
-  let botPort = 9000
-  pure Config {..}
-
 -------------------------------------------------------------------------------
 -- Provider Accounts
 

services/brig/test/integration/Federation/User.hs

@@ -0,0 +1,65 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2020 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Federation.User where
+
+import Bilge (Http, Manager)
+import qualified Brig.Options as BrigOpts
+import Brig.Types
+import Data.Handle
+import Data.Qualified
+import Imports
+import Test.Tasty
+import Test.Tasty.HUnit
+import Util
+import Util.Options (Endpoint)
+
+-- NOTE: These federation tests require deploying two sets of (some) services
+-- This might be best left to a kubernetes setup.
+--
+-- While individual functions can and should be tested in a more unit-testy way,
+-- these more end-to-end integration test serve as a way to test the overall
+-- network flow
+--
+spec :: BrigOpts.Opts -> Manager -> Brig -> Endpoint -> Brig -> IO TestTree
+spec _brigOpts mg brig _federator brigTwo =
+  pure $
+    testGroup
+      "brig-federation-user"
+      [ test mg "lookup user by qualified handle on remote backend" $ testHandleLookup brig brigTwo
+      ]
+
+testHandleLookup :: Brig -> Brig -> Http ()
+testHandleLookup brig brigTwo = do
+  -- Create a user on the "other side" using an internal brig endpoint from a
+  -- second brig instance in backendTwo (in another namespace in kubernetes)
+  u <- randomUser brigTwo
+  h <- randomHandle
+  void $ putHandle brigTwo (userId u) h
+  self <- selfUser <$> getSelfProfile brigTwo (userId u)
+  let handle = fromJust (userHandle self)
+  liftIO $ assertEqual "creating user with handle should return handle" h (fromHandle handle)
+  let domain = qDomain $ userQualifiedId self
+  resultTwo <- userHandleId <$> getUserInfoFromHandle brigTwo domain handle
+  -- query the local-namespace brig for a user sitting on the other backend
+  -- which should involve the following network traffic:
+  --
+  -- brig-integration -> brig -> federator -> fed2-federator -> fed2-brig
+  -- (and back)
+  result <- userHandleId <$> getUserInfoFromHandle brig domain handle
+  liftIO $ assertEqual "remote handle lookup via federator should work in the happy case" result (Qualified (userId u) domain)
+  liftIO $ assertEqual "querying brig1 or brig2 about remote user should give same result" resultTwo result