followup: update warp

libs/http2-manager/src/HTTP2/Client/Manager/Internal.hs

@@ -15,6 +15,7 @@ import Control.Monad
 import Control.Monad.IO.Class
 import Data.ByteString
 import qualified Data.ByteString as BS
+import qualified Data.ByteString.Char8 as C8
 import Data.IORef
 import Data.Map
 import qualified Data.Map as Map
@@ -291,9 +292,9 @@ startPersistentHTTP2Connection ::
 startPersistentHTTP2Connection ctx (tlsEnabled, hostname, port) cl removeTrailingDot tcpConnectTimeout sendReqMVar = do
   liveReqs <- newIORef mempty
   let clientConfig =
-        HTTP2.ClientConfig
+        HTTP2.defaultClientConfig
           { HTTP2.scheme = if tlsEnabled then "https" else "http",
-            HTTP2.authority = hostname,
+            HTTP2.authority = C8.unpack hostname,
             HTTP2.cacheLimit = cl
           }
       -- Sends error to requests which show up too late, i.e. after the
@@ -333,7 +334,7 @@ startPersistentHTTP2Connection ctx (tlsEnabled, hostname, port) cl removeTrailin
     bracket connectTCPWithTimeout NS.close $ \sock -> do
       bracket (mkTransport sock transportConfig) cleanupTransport $ \transport ->
         bracket (allocHTTP2Config transport) HTTP2.freeSimpleConfig $ \http2Cfg -> do
-          let runAction = HTTP2.run clientConfig http2Cfg $ \sendReq -> do
+          let runAction = HTTP2.run clientConfig http2Cfg $ \sendReq _aux -> do
                 handleRequests liveReqs sendReq
           -- Any request threads still hanging about after 'runAction' finishes
           -- are canceled with 'ConnectionAlreadyClosed'.
@@ -451,6 +452,9 @@ allocHTTP2Config (SecureTransport ssl) = do
               error "openssl: SSL.read returned more bytes than asked for, this is probably a bug"
           | otherwise ->
               readData (acc <> chunk) (n - chunkLen)
+  let s = fromMaybe (error "http2-manager: SSL without socket") $ SSL.sslSocket ssl
+  mysa <- NS.getSocketName s
+  peersa <- NS.getPeerName s
 
   pure
     HTTP2.Config
@@ -459,5 +463,7 @@ allocHTTP2Config (SecureTransport ssl) = do
         HTTP2.confSendAll = SSL.write ssl,
         HTTP2.confReadN = readData mempty,
         HTTP2.confPositionReadMaker = HTTP2.defaultPositionReadMaker,
-        HTTP2.confTimeoutManager = timmgr
+        HTTP2.confTimeoutManager = timmgr,
+        HTTP2.confMySockAddr = mysa,
+        HTTP2.confPeerSockAddr = peersa
       }

libs/http2-manager/test/Test/HTTP2/Client/ManagerSpec.hs

@@ -23,7 +23,7 @@ import qualified Data.ByteString.Lazy.Char8 as LBS
 import Data.IORef
 import Data.Map (Map)
 import qualified Data.Map as Map
-import Data.Maybe (isJust)
+import Data.Maybe (fromMaybe, isJust)
 import Data.Streaming.Network (bindPortTCP, bindRandomPortTCP)
 import Data.Unique
 import Foreign.Marshal.Alloc (mallocBytes)
@@ -33,8 +33,10 @@ import HTTP2.Client.Manager.Internal
 import Network.HTTP.Types
 import qualified Network.HTTP2.Client as Client
 import qualified Network.HTTP2.Client as HTTP2
+import Network.HTTP2.Server (defaultServerConfig)
 import qualified Network.HTTP2.Server as Server
 import Network.Socket
+import qualified Network.Socket as NS
 import qualified OpenSSL.Session as SSL
 import System.Random (randomRIO)
 import qualified System.TimeManager
@@ -293,14 +295,20 @@ allocServerConfig (Right ssl) = do
               error "openssl: SSL.read returned more bytes than asked for, this is probably a bug"
           | otherwise ->
               readData (prevChunk <> chunk) (n - chunkLen)
+
+  let s = fromMaybe (error "http2-manager: SSL without socket") $ SSL.sslSocket ssl
+  mysa <- NS.getSocketName s
+  peersa <- NS.getPeerName s
   pure
     Server.Config
       { Server.confWriteBuffer = buf,
         Server.confBufferSize = bufsize,
         Server.confSendAll = SSL.write ssl,
         Server.confReadN = readData mempty,
         Server.confPositionReadMaker = Server.defaultPositionReadMaker,
-        Server.confTimeoutManager = timmgr
+        Server.confTimeoutManager = timmgr,
+        Server.confMySockAddr = mysa,
+        Server.confPeerSockAddr = peersa
       }
 
 testServerOnSocket :: Maybe SSL.SSLContext -> Socket -> IORef Int -> IORef (Map Unique (Async ())) -> IO ()
@@ -322,7 +330,7 @@ testServerOnSocket mCtx listenSock connsCounter conns = do
         cleanup cfg = do
           Server.freeSimpleConfig cfg `finally` (shutdownSSL `finally` close sock)
     thread <- async $ bracket (allocServerConfig serverCfgParam) cleanup $ \cfg -> do
-      Server.run cfg testServer `finally` modifyIORef conns (Map.delete connKey)
+      Server.run defaultServerConfig cfg testServer `finally` modifyIORef conns (Map.delete connKey)
     modifyIORef conns $ Map.insert connKey thread
 
 testServer :: Server.Request -> Server.Aux -> (Server.Response -> [Server.PushPromise] -> IO ()) -> IO ()

libs/wire-api-federation/src/Wire/API/Federation/Client.hs

@@ -120,7 +120,7 @@ instance VersionedMonad Version (FederatorClient c) where
 liftCodensity :: Codensity IO a -> FederatorClient c a
 liftCodensity = FederatorClient . lift . lift . lift
 
-headersFromTable :: HTTP2.HeaderTable -> [HTTP.Header]
+headersFromTable :: HTTP2.TokenHeaderTable -> [HTTP.Header]
 headersFromTable (headerList, _) = flip map headerList $ first HTTP2.tokenKey
 
 -- This opens a new http2 connection. Using a http2-manager leads to this problem https://wearezeta.atlassian.net/browse/WPB-4787

nix/haskell-pins.nix

@@ -258,7 +258,20 @@ let
         hash = "sha256-E35PVxi/4iJFfWts3td52KKZKQt4dj9KFP3SvWG77Cc=";
       };
     };
+
+    # open PR https://github.com/yesodweb/wai/pull/958 for sending connection: close when closing connection
+    warp = {
+      packages.warp = "warp";
+      src = pkgs.fetchFromGitHub {
+        owner = "yesodweb";
+        repo = "wai";
+        rev = "8b20c9db265a202a2c7ba2a9ec8786a1ee59957b";
+        hash = "sha256-fKUSiRl38FKY1gFSmbksktoqoLfQrDxRRWEh4k+RRW4=";
+      };
+    };
+
   };
+
   hackagePins = {
     # Major re-write upstream, we should get rid of this dependency rather than
     # adapt to upstream, this will go away when completing servantification.
@@ -267,16 +280,46 @@ let
       sha256 = "sha256-DSMckKIeVE/buSMg8Mq+mUm1bYPYB7veA11Ns7vTBbc=";
     };
 
+    # start pinned dependencies for http2
+
+    # this contains an important fix to the initialization of the window size
+    # and should be switched to upstream as soon as we can
     http2 = {
-      version = "4.1.4";
-      sha256 = "sha256-r4Bu0vourKMkBO1cPeJVszSbAqHopmkv9EeTHcaTfuo=";
+      version = "5.2.5";
+      sha256 = "sha256-FCd4lPydwWqm2lrhgYtPW+BuXGqmmA8KFrB87SYEowY=";
     };
 
-    # warp is not compatible with 
-    warp = {
-      version = "3.3.30";
-      sha256 = "sha256-VrK27a2wFtezh9qabcXGe2tw9EwBmI8mKwmpCtXq9rc=";
+    http-semantics = {
+      version = "0.1.2";
+      sha256 = "sha256-S4rGBCIKVPpLPumLcVzrPONrbWm8VBizqxI3dXNIfr0=";
     };
+
+    network-run = {
+      version = "0.3.0";
+      sha256 = "sha256-FP2GZKwacC+TLLwEIVgKBtnKplYPf5xOIjDfvlbQV0o=";
+    };
+    time-manager = {
+      version = "0.1.0";
+      sha256 = "sha256-WRe9LZrOIPJVBFk0vMN2IMoxgP0a0psQCiCiOFWJc74=";
+    };
+    auto-update = {
+      version = "0.2.0";
+      sha256 = "sha256-d/0IDjaaCLz8tlx88z8Ew8ol9PrSRPVWaUwTbim70yE=";
+    };
+
+    network-control = {
+      version = "0.1.0";
+      sha256 = "sha256-D6pKb6+0Pr08FnObGbXBVMv04ys3N731p7U+GYH1oEg=";
+    };
+    # end pinned dependencies for http2
+
+    # pinned for warp
+    warp-tls = {
+      version = "3.4.5";
+      sha256 = "sha256-3cDi/+n7wHfcWT/iFWAsGdLYXtKYXmvzolDt+ACJnaM=";
+    };
+    # end pinned for warp
+
     # PR: https://github.com/wireapp/wire-server/pull/4027
     HsOpenSSL = {
       version = "0.11.7.7";
@@ -311,11 +354,12 @@ let
     gitPins;
   # AttrSet
   hackagePackages = lib.attrsets.mapAttrs
-    (pkg: { version, sha256 }:
+    (pkg: args:
       hself.callHackageDirect
         {
-          ver = version;
-          inherit pkg sha256;
+          ver = args.version;
+          sha256 = args.sha256 or "";
+          inherit pkg;
         }
         { }
     )

nix/manual-overrides.nix

@@ -70,20 +70,8 @@ hself: hsuper: {
   tls = hsuper.tls_2_0_5;
   tls-session-manager = hsuper.tls-session-manager_0_0_5;
 
-  # for warp (and its transitive deps)
-  # we have a PR open https://github.com/yesodweb/wai/pull/958
-  # unfortunately, because of breakage in http2, our fork has moved beyond what 
-  # we can use in wire itself, hence the patch
-  # the version of warp is pinned in ./haskell-pins.nix
-  warp = hlib.addTestToolDepends
-    (hlib.appendPatches hsuper.warp [
-      (fetchpatch {
-        url = "https://github.com/yesodweb/wai/commit/ef993a357822d9bc2a2040afcb656b31c378491c.patch";
-        stripLen = 1;
-        sha256 = "sha256-rv/ujqyBmpsChQg2uS3/HUgQZCA3SzBiF8kUnZJN0xs=";
-      })
-    ]) [ curl ];
-  # end for warp
+  # warp requires curl in its testsuite
+  warp = hlib.addTestToolDepends hsuper.warp [ curl ];
 
   # -----------------
   # flags and patches