Federation: Implement ID mapping (galley)

docs/reference/cassandra-schema.cql

@@ -330,6 +330,25 @@ CREATE TABLE galley_test.legalhold_service (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+CREATE TABLE galley_test.id_mapping (
+    mapped_id uuid PRIMARY KEY,
+    remote_domain text,
+    remote_id uuid
+) WITH bloom_filter_fp_chance = 0.1
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
 CREATE TABLE galley_test.billing_team_member (
     team uuid,
     user uuid,

libs/galley-types/galley-types.cabal

@@ -4,7 +4,7 @@ cabal-version: 1.12
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: 73ad5a5126cffda9d353014c94e6f72b68f8dfbe7ecad75a7f03f55f13e06d7b
+-- hash: b41772acd74181a186da3c81fa43def5f82ca80b35c55f158ae40376fa8097ff
 
 name:           galley-types
 version:        0.81.0
@@ -24,6 +24,7 @@ library
       Galley.Types.Bot.Service
       Galley.Types.Conversations.Members
       Galley.Types.Conversations.Roles
+      Galley.Types.IdMapping
       Galley.Types.Teams
       Galley.Types.Teams.Intra
       Galley.Types.Teams.SearchVisibility

libs/galley-types/src/Galley/Types/IdMapping.hs

@@ -0,0 +1,68 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2020 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Galley.Types.IdMapping
+  ( PostIdMappingRequest (..),
+    mkPostIdMappingRequest,
+    PostIdMappingResponse (..),
+  )
+where
+
+import Data.Aeson
+import Data.Coerce (coerce)
+import Data.Id (Id, Mapped, Remote)
+import Data.Qualified (Qualified)
+import Imports
+
+-- | Request used for inter-service communication between Galley and Brig to ensure that ID
+-- mappings discovered in one service are also known in the other.
+--
+-- The type of Id (user or conversation) doesn't matter, since it's all the same table.
+-- Therefore, we use @()@.
+newtype PostIdMappingRequest = PostIdMappingRequest
+  {reqQualifiedId :: Qualified (Id (Remote ()))}
+  deriving stock (Eq, Show)
+
+mkPostIdMappingRequest :: Qualified (Id (Remote a)) -> PostIdMappingRequest
+mkPostIdMappingRequest = PostIdMappingRequest . coerce
+
+instance ToJSON PostIdMappingRequest where
+  toJSON (PostIdMappingRequest qualifiedId) =
+    object ["qualified_id" .= qualifiedId]
+
+instance FromJSON PostIdMappingRequest where
+  parseJSON = withObject "PostIdMappingRequest" $ \o ->
+    PostIdMappingRequest <$> o .: "qualified_id"
+
+-- | Response used for inter-service communication between Galley and Brig to ensure that ID
+-- mappings discovered in one service are also known in the other.
+--
+-- The type of Id (user or conversation) doesn't matter, since it's all the same table.
+-- Therefore, we use @()@.
+newtype PostIdMappingResponse = PostIdMappingResponse
+  {resMappedId :: Id (Mapped ())}
+  deriving stock (Eq, Show)
+
+instance ToJSON PostIdMappingResponse where
+  toJSON (PostIdMappingResponse mappedId) =
+    object ["mapped_id" .= mappedId]
+
+instance FromJSON PostIdMappingResponse where
+  parseJSON = withObject "PostIdMappingResponse" $ \o ->
+    PostIdMappingResponse <$> o .: "mapped_id"

libs/galley-types/test/unit/Test/Galley/Types.hs

@@ -1,6 +1,5 @@
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
 {-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE TypeApplications #-}
 {-# OPTIONS_GHC -Wno-orphans #-}
 
 -- This file is part of the Wire Server implementation.
@@ -24,6 +23,7 @@ module Test.Galley.Types where
 
 import Control.Lens
 import Data.Set hiding (drop)
+import Galley.Types.IdMapping (PostIdMappingRequest (..), PostIdMappingResponse (..))
 import Galley.Types.Teams
 import Imports
 import Test.Galley.Roundtrip (testRoundTrip)
@@ -54,7 +54,9 @@ tests =
         -- accordingly.  Just maintain the property that adding a new feature name will break
         -- this test, and force future develpers to consider what permissions they want to set.
         assertBool "all covered" (all (roleHasPerm RoleExternalPartner) (ViewTeamFeature <$> [minBound ..])),
-      testRoundTrip @FeatureFlags
+      testRoundTrip @FeatureFlags,
+      testRoundTrip @PostIdMappingRequest,
+      testRoundTrip @PostIdMappingRequest
     ]
 
 instance Arbitrary FeatureFlags where
@@ -63,3 +65,7 @@ instance Arbitrary FeatureFlags where
       <$> QC.elements [minBound ..]
       <*> QC.elements [minBound ..]
       <*> QC.elements [minBound ..]
+
+deriving newtype instance Arbitrary PostIdMappingRequest
+
+deriving newtype instance Arbitrary PostIdMappingResponse

libs/types-common/src/Data/IdMapping.hs

@@ -19,11 +19,15 @@
 
 module Data.IdMapping where
 
+import Data.Aeson ((.=), ToJSON (toJSON), object)
 import Data.Id
 import Data.Qualified
 import Imports
 import Test.QuickCheck (Arbitrary (arbitrary), oneof)
 
+----------------------------------------------------------------------
+-- MappedOrLocalId
+
 data MappedOrLocalId a
   = Mapped (IdMapping a)
   | Local (Id a)
@@ -32,19 +36,31 @@ data MappedOrLocalId a
 opaqueIdFromMappedOrLocal :: MappedOrLocalId a -> Id (Opaque a)
 opaqueIdFromMappedOrLocal = \case
   Local localId -> makeIdOpaque localId
-  Mapped IdMapping {idMappingLocal} -> makeMappedIdOpaque idMappingLocal
+  Mapped IdMapping {_imMappedId} -> makeMappedIdOpaque _imMappedId
 
 partitionMappedOrLocalIds :: Foldable f => f (MappedOrLocalId a) -> ([Id a], [IdMapping a])
 partitionMappedOrLocalIds = foldMap $ \case
   Mapped mapping -> (mempty, [mapping])
   Local localId -> ([localId], mempty)
 
+----------------------------------------------------------------------
+-- IdMapping
+
 data IdMapping a = IdMapping
-  { idMappingLocal :: Id (Mapped a),
-    idMappingGlobal :: Qualified (Id (Remote a))
+  { _imMappedId :: Id (Mapped a),
+    _imQualifiedId :: Qualified (Id (Remote a))
   }
   deriving stock (Eq, Ord, Show)
 
+-- Don't add a FromJSON instance!
+-- We don't want to just accept mappings we didn't create ourselves.
+instance ToJSON (IdMapping a) where
+  toJSON IdMapping {_imMappedId, _imQualifiedId} =
+    object
+      [ "mapped_id" .= _imMappedId,
+        "qualified_id" .= _imQualifiedId
+      ]
+
 ----------------------------------------------------------------------
 -- ARBITRARY
 

services/brig/src/Brig/API/Client.hs

@@ -76,9 +76,9 @@ lookupClient mappedOrLocalUserId clientId =
   case mappedOrLocalUserId of
     Local u ->
       lift $ lookupLocalClient u clientId
-    Mapped IdMapping {idMappingLocal} ->
+    Mapped IdMapping {_imMappedId} ->
       -- FUTUREWORK(federation, #1271): look up remote clients
-      throwE $ ClientUserNotFound (makeMappedIdOpaque idMappingLocal)
+      throwE $ ClientUserNotFound (makeMappedIdOpaque _imMappedId)
 
 lookupLocalClient :: UserId -> ClientId -> AppIO (Maybe Client)
 lookupLocalClient = Data.lookupClient
@@ -87,9 +87,9 @@ lookupClients :: MappedOrLocalId Id.U -> ExceptT ClientError AppIO [Client]
 lookupClients = \case
   Local u ->
     lift $ lookupLocalClients u
-  Mapped IdMapping {idMappingLocal} ->
+  Mapped IdMapping {_imMappedId} ->
     -- FUTUREWORK(federation, #1271): look up remote clients
-    throwE $ ClientUserNotFound (makeMappedIdOpaque idMappingLocal)
+    throwE $ ClientUserNotFound (makeMappedIdOpaque _imMappedId)
 
 lookupLocalClients :: UserId -> AppIO [Client]
 lookupLocalClients = Data.lookupClients
@@ -160,9 +160,9 @@ claimPrekeyBundle :: MappedOrLocalId Id.U -> AppIO PrekeyBundle
 claimPrekeyBundle = \case
   Local localUser ->
     claimLocalPrekeyBundle localUser
-  Mapped IdMapping {idMappingLocal} ->
+  Mapped IdMapping {_imMappedId} ->
     -- FUTUREWORK(federation, #1272): claim keys from other backend
-    pure $ PrekeyBundle (makeMappedIdOpaque idMappingLocal) []
+    pure $ PrekeyBundle (makeMappedIdOpaque _imMappedId) []
 
 claimLocalPrekeyBundle :: UserId -> AppIO PrekeyBundle
 claimLocalPrekeyBundle u = do

services/brig/src/Brig/API/Connection.hs

@@ -48,7 +48,7 @@ import qualified Brig.User.Event.Log as Log
 import Control.Error
 import Control.Lens (view)
 import Data.Id as Id
-import Data.IdMapping (IdMapping (IdMapping, idMappingLocal), MappedOrLocalId (Local, Mapped))
+import Data.IdMapping (IdMapping (IdMapping, _imMappedId), MappedOrLocalId (Local, Mapped))
 import Data.Range
 import qualified Data.Set as Set
 import Galley.Types (ConvType (..), cnvType)
@@ -66,9 +66,9 @@ createConnection self req conn = do
   resolveOpaqueUserId (crUser req) >>= \case
     Local u ->
       createConnectionToLocalUser self u req conn
-    Mapped IdMapping {idMappingLocal} ->
+    Mapped IdMapping {_imMappedId} ->
       -- FUTUREWORK(federation, #1262): allow creating connections to remote users
-      throwE $ InvalidUser (makeMappedIdOpaque idMappingLocal)
+      throwE $ InvalidUser (makeMappedIdOpaque _imMappedId)
 
 createConnectionToLocalUser ::
   UserId ->

services/brig/src/Brig/API/Error.hs

@@ -28,7 +28,7 @@ import Data.ByteString.Conversion
 import Data.Domain (Domain)
 import qualified Data.HashMap.Strict as HashMap
 import Data.Id (idToText)
-import Data.IdMapping (IdMapping (IdMapping, idMappingGlobal, idMappingLocal))
+import Data.IdMapping (IdMapping (IdMapping, _imMappedId, _imQualifiedId))
 import Data.List.NonEmpty (NonEmpty)
 import Data.Qualified (renderQualifiedId)
 import Data.String.Conversions (cs)
@@ -484,8 +484,8 @@ federationNotImplemented qualified =
   where
     idType = cs (show (typeRep @a))
     rendered = LT.intercalate ", " . toList . fmap (LT.fromStrict . renderMapping) $ qualified
-    renderMapping IdMapping {idMappingLocal, idMappingGlobal} =
-      idToText idMappingLocal <> " -> " <> renderQualifiedId idMappingGlobal
+    renderMapping IdMapping {_imMappedId, _imQualifiedId} =
+      idToText _imMappedId <> " -> " <> renderQualifiedId _imQualifiedId
 
 -- (the tautological constraint in the type signature is added so that once we remove the
 -- feature, ghc will guide us here.)

services/galley/galley.cabal

@@ -4,7 +4,7 @@ cabal-version: 1.12
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: 0078da3c962467f70b278c32cbc35b9daca407ca541f11862b2262160c86c27b
+-- hash: d61c32c046efc149b7de08f80cc5f16ac31a6c90f59d51f04b53949da3a3c1b8
 
 name:           galley
 version:        0.83.0
@@ -29,6 +29,7 @@ library
       Galley.API.Create
       Galley.API.CustomBackend
       Galley.API.Error
+      Galley.API.IdMapping
       Galley.API.Internal
       Galley.API.LegalHold
       Galley.API.Mapping
@@ -43,6 +44,7 @@ library
       Galley.Aws
       Galley.Data
       Galley.Data.CustomBackend
+      Galley.Data.IdMapping
       Galley.Data.Instances
       Galley.Data.LegalHold
       Galley.Data.Queries
@@ -54,6 +56,7 @@ library
       Galley.External
       Galley.External.LegalHoldService
       Galley.Intra.Client
+      Galley.Intra.IdMapping
       Galley.Intra.Journal
       Galley.Intra.Push
       Galley.Intra.Spar
@@ -186,6 +189,7 @@ executable galley-integration
   other-modules:
       API
       API.CustomBackend
+      API.IdMapping
       API.MessageTimer
       API.Roles
       API.SQS
@@ -340,6 +344,7 @@ executable galley-schema
       V42_TeamFeatureValidateSamlEmails
       V43_TeamFeatureDigitalSignatures
       V44_AddRemoteIdentifiers
+      V45_AddFederationIdMapping
       Paths_galley
   hs-source-dirs:
       schema/src

services/galley/schema/src/Main.hs

@@ -47,6 +47,7 @@ import qualified V41_TeamNotificationQueue
 import qualified V42_TeamFeatureValidateSamlEmails
 import qualified V43_TeamFeatureDigitalSignatures
 import qualified V44_AddRemoteIdentifiers
+import qualified V45_AddFederationIdMapping
 
 main :: IO ()
 main = do
@@ -79,7 +80,8 @@ main = do
       V41_TeamNotificationQueue.migration,
       V42_TeamFeatureValidateSamlEmails.migration,
       V43_TeamFeatureDigitalSignatures.migration,
-      V44_AddRemoteIdentifiers.migration
+      V44_AddRemoteIdentifiers.migration,
+      V45_AddFederationIdMapping.migration
       -- When adding migrations here, don't forget to update
       -- 'schemaVersion' in Galley.Data
     ]

services/galley/schema/src/V45_AddFederationIdMapping.hs

@@ -0,0 +1,37 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2020 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module V45_AddFederationIdMapping
+  ( migration,
+  )
+where
+
+import Cassandra.Schema
+import Imports
+import Text.RawString.QQ
+
+migration :: Migration
+migration = Migration 45 "Add feature flag for validation of saml emails" $ do
+  schema'
+    [r|
+        CREATE TABLE id_mapping (
+            mapped_id uuid PRIMARY KEY,
+            remote_id uuid,
+            remote_domain text,
+        ) WITH compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+           AND gc_grace_seconds = 864000;
+    |]