Merge remote-tracking branch 'origin/develop' into SQSERVICES-428-lh-…

…group-convs

hack/helm_vars/wire-server/values.yaml

@@ -31,9 +31,7 @@ brig:
   replicaCount: 1
   imagePullPolicy: Always
   resources:
-    requests:
-      memory: 128Mi
-      cpu: 500m
+    requests: {}
     limits:
       memory: 512Mi
       cpu: 500m
@@ -110,9 +108,7 @@ cannon:
   replicaCount: 2
   imagePullPolicy: Always
   resources:
-    requests:
-      memory: 512Mi
-      cpu: 500m
+    requests: {}
     limits:
       memory: 512Mi
       cpu: 500m
@@ -121,9 +117,7 @@ cargohold:
   replicaCount: 1
   imagePullPolicy: Always
   resources:
-    requests:
-      memory: 128Mi
-      cpu: 100m
+    requests: {}
     limits:
       memory: 512Mi
       cpu: 500m
@@ -164,9 +158,7 @@ gundeck:
   replicaCount: 1
   imagePullPolicy: Always
   resources:
-    requests:
-      memory: 512Mi
-      cpu: 500m
+    requests: {}
     limits:
       memory: 1024Mi
       cpu: 1000m
@@ -217,9 +209,7 @@ spar:
   replicaCount: 1
   imagePullPolicy: Always
   resources:
-    requests:
-      memory: 512Mi
-      cpu: 500m
+    requests: {}
     limits:
       memory: 1024Mi
       cpu: 1000m

libs/wire-api-federation/src/Wire/API/Federation/API/Brig.hs

@@ -29,11 +29,13 @@ import Test.QuickCheck (Arbitrary)
 import Wire.API.Arbitrary (GenericUniform (..))
 import Wire.API.Federation.Client (FederationClientFailure, FederatorClient)
 import qualified Wire.API.Federation.GRPC.Types as Proto
+import Wire.API.Federation.Util.Aeson (CustomEncoded (..))
 import Wire.API.Message (UserClients)
 import Wire.API.User (UserProfile)
-import Wire.API.User.Client (UserClientPrekeyMap)
+import Wire.API.User.Client (PubClient, UserClientPrekeyMap)
 import Wire.API.User.Client.Prekey (ClientPrekey, PrekeyBundle)
 import Wire.API.User.Search
+import Wire.API.UserMap (UserMap)
 
 newtype SearchRequest = SearchRequest {term :: Text}
   deriving (Show, Eq, Generic, Typeable)
@@ -82,9 +84,21 @@ data Api routes = Api
         -- FUTUREWORK(federation): do we want to perform some type-level validation like length checks?
         -- (handles can be up to 256 chars currently)
         :> ReqBody '[JSON] SearchRequest
-        :> Post '[JSON] (SearchResult Contact)
+        :> Post '[JSON] (SearchResult Contact),
+    getUserClients ::
+      routes
+        :- "federation"
+        :> "get-user-clients"
+        :> ReqBody '[JSON] GetUserClients
+        :> Post '[JSON] (UserMap (Set PubClient))
   }
   deriving (Generic)
 
+newtype GetUserClients = GetUserClients
+  { gucUsers :: [UserId]
+  }
+  deriving stock (Eq, Show, Generic)
+  deriving (ToJSON, FromJSON) via (CustomEncoded GetUserClients)
+
 clientRoutes :: (MonadError FederationClientFailure m, MonadIO m) => Api (AsClientT (FederatorClient 'Proto.Brig m))
 clientRoutes = genericClient

libs/wire-api/package.yaml

@@ -25,6 +25,7 @@ library:
   - case-insensitive
   - cassandra-util
   - cassava >= 0.5
+  - cereal
   - cookie
   - cryptonite
   - currency-codes >=2.0

libs/wire-api/src/Wire/API/Message.hs

@@ -21,12 +21,14 @@
 module Wire.API.Message
   ( -- * Message
     NewOtrMessage (..),
+    protoToNewOtrMessage,
 
     -- * Priority
     Priority (..),
 
     -- * Recipients
     OtrRecipients (..),
+    protoFromOtrRecipients,
     UserClientMap (..),
 
     -- * Filter
@@ -44,18 +46,25 @@ module Wire.API.Message
   )
 where
 
-import Control.Lens ((?~))
+import Control.Lens (view, (?~))
 import qualified Data.Aeson as A
+import qualified Data.ByteString.Base64 as B64
 import Data.CommaSeparatedList (CommaSeparatedList (fromCommaSeparatedList))
 import Data.Id
 import Data.Json.Util
+import qualified Data.Map.Strict as Map
+import qualified Data.ProtocolBuffers as Protobuf
 import Data.Schema
+import Data.Serialize (runGetLazy)
 import qualified Data.Set as Set
 import qualified Data.Swagger as S
 import qualified Data.Swagger.Build.Api as Doc
+import Data.Text.Encoding (decodeUtf8, encodeUtf8)
 import Imports
 import Servant (FromHttpApiData (..))
 import Wire.API.Arbitrary (Arbitrary (..), GenericUniform (..))
+import qualified Wire.API.Message.Proto as Proto
+import Wire.API.ServantProto (FromProto (..))
 import Wire.API.User.Client (UserClientMap (..), UserClients (..), modelOtrClientMap, modelUserClients)
 
 --------------------------------------------------------------------------------
@@ -115,6 +124,25 @@ instance ToSchema NewOtrMessage where
         <*> newOtrData .= opt (field "data" schema)
         <*> newOtrReportMissing .= opt (field "report_missing" (array schema))
 
+instance FromProto NewOtrMessage where
+  fromProto bs = protoToNewOtrMessage <$> runGetLazy Protobuf.decodeMessage bs
+
+protoToNewOtrMessage :: Proto.NewOtrMessage -> NewOtrMessage
+protoToNewOtrMessage msg =
+  NewOtrMessage
+    { newOtrSender = Proto.toClientId (view Proto.newOtrMessageSender msg),
+      newOtrRecipients = protoToOtrRecipients (view Proto.newOtrMessageRecipients msg),
+      newOtrNativePush = view Proto.newOtrMessageNativePush msg,
+      newOtrTransient = view Proto.newOtrMessageTransient msg,
+      newOtrData = toBase64Text <$> view Proto.newOtrMessageData msg,
+      newOtrNativePriority = protoToPriority <$> view Proto.newOtrMessageNativePriority msg,
+      newOtrReportMissing = protoToReportMissing $ view Proto.newOtrMessageReportMissing msg
+    }
+
+protoToReportMissing :: [Proto.UserId] -> Maybe [UserId]
+protoToReportMissing [] = Nothing
+protoToReportMissing us = Just $ view Proto.userId <$> us
+
 --------------------------------------------------------------------------------
 -- Priority
 
@@ -149,6 +177,10 @@ instance ToSchema Priority where
           element "high" HighPriority
         ]
 
+protoToPriority :: Proto.Priority -> Priority
+protoToPriority Proto.LowPriority = LowPriority
+protoToPriority Proto.HighPriority = HighPriority
+
 --------------------------------------------------------------------------------
 -- Recipients
 
@@ -166,6 +198,32 @@ modelOtrRecipients = Doc.defineModel "OtrRecipients" $ do
   Doc.property "" (Doc.ref modelOtrClientMap) $
     Doc.description "Mapping of user IDs to 'OtrClientMap's."
 
+protoToOtrRecipients :: [Proto.UserEntry] -> OtrRecipients
+protoToOtrRecipients =
+  OtrRecipients . UserClientMap
+    . foldl' userEntries mempty
+  where
+    userEntries :: Map UserId (Map ClientId Text) -> Proto.UserEntry -> Map UserId (Map ClientId Text)
+    userEntries acc x =
+      let u = view Proto.userEntryId x
+          c = view Proto.userEntryClients x
+          m = foldl' clientEntries mempty c
+       in Map.insert (view Proto.userId u) m acc
+    clientEntries acc x =
+      let c = Proto.toClientId $ view Proto.clientEntryId x
+          t = toBase64Text $ view Proto.clientEntryMessage x
+       in Map.insert c t acc
+
+protoFromOtrRecipients :: OtrRecipients -> [Proto.UserEntry]
+protoFromOtrRecipients rcps =
+  let m = userClientMap (otrRecipientsMap rcps)
+   in map mkProtoRecipient (Map.toList m)
+  where
+    mkProtoRecipient (usr, clts) =
+      let xs = map mkClientEntry (Map.toList clts)
+       in Proto.userEntry (Proto.fromUserId usr) xs
+    mkClientEntry (clt, t) = Proto.clientEntry (Proto.fromClientId clt) (fromBase64Text t)
+
 --------------------------------------------------------------------------------
 -- Filter
 
@@ -250,3 +308,12 @@ instance FromHttpApiData ReportMissing where
     "true" -> Right ReportMissingAll
     "false" -> Right $ ReportMissingList mempty
     list -> ReportMissingList . Set.fromList . fromCommaSeparatedList <$> parseQueryParam list
+
+--------------------------------------------------------------------------------
+-- Utilities
+
+fromBase64Text :: Text -> ByteString
+fromBase64Text = B64.decodeLenient . encodeUtf8
+
+toBase64Text :: ByteString -> Text
+toBase64Text = decodeUtf8 . B64.encode

libs/wire-api/src/Wire/API/Message/Proto.hs

@@ -35,11 +35,7 @@ module Wire.API.Message.Proto
     userEntry,
     userEntryId,
     userEntryClients,
-    toOtrRecipients,
-    fromOtrRecipients,
     Priority (..),
-    toPriority,
-    fromPriority,
     NewOtrMessage,
     newOtrMessage,
     newOtrMessageSender,
@@ -49,21 +45,14 @@ module Wire.API.Message.Proto
     newOtrMessageData,
     newOtrMessageTransient,
     newOtrMessageReportMissing,
-    toNewOtrMessage,
   )
 where
 
-import Control.Lens (view)
-import qualified Data.ByteString.Base64 as B64
 import qualified Data.Id as Id
-import qualified Data.Map.Strict as Map
 import Data.ProtocolBuffers
-import Data.Text.Encoding (decodeUtf8, encodeUtf8)
 import qualified Data.Text.Lazy as Text
 import Data.Text.Lazy.Read (hexadecimal)
 import Imports
-import qualified Wire.API.Message as Msg
-import qualified Wire.API.User.Client as Client
 
 --------------------------------------------------------------------------------
 -- UserId
@@ -163,32 +152,6 @@ userEntryId f c = (\x -> c {_userId = x}) <$> field f (_userId c)
 userEntryClients :: Functor f => ([ClientEntry] -> f [ClientEntry]) -> UserEntry -> f UserEntry
 userEntryClients f c = (\x -> c {_userVal = x}) <$> field f (_userVal c)
 
-toOtrRecipients :: [UserEntry] -> Msg.OtrRecipients
-toOtrRecipients =
-  Msg.OtrRecipients . Client.UserClientMap
-    . foldl' userEntries mempty
-  where
-    userEntries :: Map Id.UserId (Map Id.ClientId Text) -> UserEntry -> Map Id.UserId (Map Id.ClientId Text)
-    userEntries acc x =
-      let u = view userEntryId x
-          c = view userEntryClients x
-          m = foldl' clientEntries mempty c
-       in Map.insert (view userId u) m acc
-    clientEntries acc x =
-      let c = toClientId $ view clientEntryId x
-          t = toBase64Text $ view clientEntryMessage x
-       in Map.insert c t acc
-
-fromOtrRecipients :: Msg.OtrRecipients -> [UserEntry]
-fromOtrRecipients rcps =
-  let m = Client.userClientMap (Msg.otrRecipientsMap rcps)
-   in map mkProtoRecipient (Map.toList m)
-  where
-    mkProtoRecipient (usr, clts) =
-      let xs = map mkClientEntry (Map.toList clts)
-       in UserEntry (putField (fromUserId usr)) (putField xs)
-    mkClientEntry (clt, t) = clientEntry (fromClientId clt) (fromBase64Text t)
-
 --------------------------------------------------------------------------------
 -- Priority
 
@@ -211,14 +174,6 @@ instance Bounded Priority where
   minBound = LowPriority
   maxBound = HighPriority
 
-toPriority :: Priority -> Msg.Priority
-toPriority LowPriority = Msg.LowPriority
-toPriority HighPriority = Msg.HighPriority
-
-fromPriority :: Msg.Priority -> Priority
-fromPriority Msg.LowPriority = LowPriority
-fromPriority Msg.HighPriority = HighPriority
-
 --------------------------------------------------------------------------------
 -- NewOtrMessage
 
@@ -273,28 +228,3 @@ newOtrMessageNativePriority f c = (\x -> c {_newOtrNativePriority = x}) <$> fiel
 
 newOtrMessageReportMissing :: Functor f => ([UserId] -> f [UserId]) -> NewOtrMessage -> f NewOtrMessage
 newOtrMessageReportMissing f c = (\x -> c {_newOtrReportMissing = x}) <$> field f (_newOtrReportMissing c)
-
-toNewOtrMessage :: NewOtrMessage -> Msg.NewOtrMessage
-toNewOtrMessage msg =
-  Msg.NewOtrMessage
-    { Msg.newOtrSender = toClientId (view newOtrMessageSender msg),
-      Msg.newOtrRecipients = toOtrRecipients (view newOtrMessageRecipients msg),
-      Msg.newOtrNativePush = view newOtrMessageNativePush msg,
-      Msg.newOtrTransient = view newOtrMessageTransient msg,
-      Msg.newOtrData = toBase64Text <$> view newOtrMessageData msg,
-      Msg.newOtrNativePriority = toPriority <$> view newOtrMessageNativePriority msg,
-      Msg.newOtrReportMissing = toReportMissing $ view newOtrMessageReportMissing msg
-    }
-
-toReportMissing :: [UserId] -> Maybe [Id.UserId]
-toReportMissing [] = Nothing
-toReportMissing us = Just $ view userId <$> us
-
---------------------------------------------------------------------------------
--- Utilities
-
-fromBase64Text :: Text -> ByteString
-fromBase64Text = B64.decodeLenient . encodeUtf8
-
-toBase64Text :: ByteString -> Text
-toBase64Text = decodeUtf8 . B64.encode

libs/wire-api/src/Wire/API/Routes/Public/Galley.hs

@@ -37,6 +37,7 @@ import Wire.API.ErrorDescription (ConversationNotFound, UnknownClient)
 import qualified Wire.API.Event.Conversation as Public
 import qualified Wire.API.Message as Public
 import Wire.API.Routes.Public (EmptyResult, ZConn, ZUser)
+import Wire.API.ServantProto (Proto)
 import qualified Wire.API.Team.Conversation as Public
 
 type ConversationResponses =
@@ -224,7 +225,7 @@ data Api routes = Api
         :> Delete '[] (EmptyResult 200),
     postOtrMessage ::
       routes
-        :- Summary "Post an encrypted message to a conversation (accepts JSON)"
+        :- Summary "Post an encrypted message to a conversation (accepts JSON or Protobuf)"
         :> Description PostOtrDescription
         :> ZUser
         :> ZConn
@@ -234,7 +235,7 @@ data Api routes = Api
         :> QueryParam "report_missing" Public.ReportMissing
         :> "otr"
         :> "messages"
-        :> ReqBody '[Servant.JSON] Public.NewOtrMessage
+        :> ReqBody '[Servant.JSON, Proto] Public.NewOtrMessage
         :> UVerb 'POST '[Servant.JSON] PostOtrResponses
   }
   deriving (Generic)
@@ -260,7 +261,9 @@ type PostOtrDescription =
   \- `ignore_missing` in the query param is the next.\n\
   \- `report_missing` in the query param has the lowest precedence.\n\
   \\n\
-  \This endpoint can lead to OtrMessageAdd event being sent to the recipients."
+  \This endpoint can lead to OtrMessageAdd event being sent to the recipients.\n\
+  \\n\
+  \**NOTE:** The protobuf definitions of the request body can be found at https://github.com/wireapp/generic-message-proto/blob/master/proto/otr.proto."
 
 swaggerDoc :: Swagger.Swagger
 swaggerDoc = toSwagger (Proxy @ServantAPI)

libs/wire-api/src/Wire/API/ServantProto.hs

@@ -0,0 +1,25 @@
+module Wire.API.ServantProto where
+
+import Data.List.NonEmpty (NonEmpty (..))
+import Imports
+import Network.HTTP.Media ((//))
+import Servant
+
+-- | Type to tell servant that it should unrender request body or render
+-- response body with Protobuf
+data Proto
+
+-- | We do not use 'Data.ProtocolBuffers.Decode' so we get a little freedom in
+-- defining separate data types which match one to one with the protobuf and the
+-- data types which we actually use in business logic. Eventually we should
+-- think of better ways of doing this, perhaps using mu-schema or proto-lens as
+-- it is fairly difficult to keep our custom data type, e.g. in
+-- Wire.API.Message.Proto in sync with the proto files.
+class FromProto a where
+  fromProto :: LByteString -> Either String a
+
+instance Accept Proto where
+  contentTypes _ = ("application" // "x-protobuf") :| []
+
+instance FromProto a => MimeUnrender Proto a where
+  mimeUnrender _ bs = fromProto bs