From 83ae70d264ef2dcd229407db9d2c47a151d030ef Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Wed, 18 May 2022 16:36:47 -0400 Subject: [PATCH 1/3] fix: adjust rcmgr limits for accelerated DHT client rt refresh The Accelerated DHT client periodically refreshes its routing table, including at startup, and if Resource Manager throttling causes the client's routing table to be incomplete, then content routing may be degraded or broken for users. This adjusts the default limits to a level that empirically doesn't cause Resource Manager throttling during initial DHT client bootstrapping. Ideally the Accelerated DHT client would handle this scenario more gracefully, but this works for now to unblock the 0.13 release. --- core/node/libp2p/rcmgr_defaults.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/core/node/libp2p/rcmgr_defaults.go b/core/node/libp2p/rcmgr_defaults.go index 11d24364149..404dd2b3e1e 100644 --- a/core/node/libp2p/rcmgr_defaults.go +++ b/core/node/libp2p/rcmgr_defaults.go @@ -38,14 +38,17 @@ func adjustedDefaultLimits(cfg config.SwarmConfig) rcmgr.DefaultLimitConfig { if cfg.ConnMgr.Type == "basic" { maxconns := cfg.ConnMgr.HighWater if 2*maxconns > defaultLimits.SystemBaseLimit.ConnsInbound { - // adjust conns to 2x to allow for two conns per peer (TCP+QUIC) + // Conns should be at least 2x larger than the high water to allow for two conns per peer (TCP+QUIC). + // + // Outbound conns are set very high to allow for the accelerated DHT client to refresh its routing table. + // Currently it does not gracefully handle RM throttling, once it does we can lower this. defaultLimits.SystemBaseLimit.ConnsInbound = logScale(2 * maxconns) - defaultLimits.SystemBaseLimit.ConnsOutbound = logScale(2 * maxconns) - defaultLimits.SystemBaseLimit.Conns = logScale(4 * maxconns) + defaultLimits.SystemBaseLimit.ConnsOutbound = logScale(64 * maxconns) + defaultLimits.SystemBaseLimit.Conns = defaultLimits.SystemBaseLimit.ConnsOutbound + defaultLimits.SystemBaseLimit.ConnsInbound - defaultLimits.SystemBaseLimit.StreamsInbound = logScale(16 * maxconns) - defaultLimits.SystemBaseLimit.StreamsOutbound = logScale(64 * maxconns) - defaultLimits.SystemBaseLimit.Streams = logScale(64 * maxconns) + defaultLimits.SystemBaseLimit.StreamsInbound = defaultLimits.SystemBaseLimit.ConnsInbound * 4 + defaultLimits.SystemBaseLimit.StreamsOutbound = defaultLimits.SystemBaseLimit.ConnsOutbound * 16 + defaultLimits.SystemBaseLimit.Streams = defaultLimits.SystemBaseLimit.Conns * 16 if 2*maxconns > defaultLimits.SystemBaseLimit.FD { defaultLimits.SystemBaseLimit.FD = logScale(2 * maxconns) From 5c7c88f4a28bf727758866e135e5ebe21ac25b5a Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Fri, 27 May 2022 10:09:23 -0400 Subject: [PATCH 2/3] Set default outbound conns unconditionally This also sets the default overall conns as a function of the outbound and inbound conns, since they are adjusted dynamically, and it makes the intention of the value clear. --- core/node/libp2p/rcmgr_defaults.go | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/core/node/libp2p/rcmgr_defaults.go b/core/node/libp2p/rcmgr_defaults.go index 404dd2b3e1e..ddc6a82da2b 100644 --- a/core/node/libp2p/rcmgr_defaults.go +++ b/core/node/libp2p/rcmgr_defaults.go @@ -25,35 +25,38 @@ func adjustedDefaultLimits(cfg config.SwarmConfig) rcmgr.DefaultLimitConfig { checkImplicitDefaults() } - // Return to use unmodified static limits based on values from go-libp2p 0.18 - // return defaultLimits - // Adjust limits // (based on https://github.com/filecoin-project/lotus/pull/8318/files) // - give it more memory, up to 4G, min of 1G // - if Swarm.ConnMgr.HighWater is too high, adjust Conn/FD/Stream limits defaultLimits := rcmgr.DefaultLimits.WithSystemMemory(.125, 1<<30, 4<<30) + // Outbound conns are set very high to allow for the accelerated DHT client to (re)load its routing table. + // Currently it doesn't gracefully handle RM throttling--once it does we can lower this. + // High outbound conn limits are considered less of a DoS risk than high inbound conn limits. + // Also note that, due to the behavior of the accelerated DHT client, we don't need many streams, just conns. + defaultLimits.SystemBaseLimit.ConnsOutbound = 65536 + // Do we need to adjust due to Swarm.ConnMgr.HighWater? if cfg.ConnMgr.Type == "basic" { maxconns := cfg.ConnMgr.HighWater if 2*maxconns > defaultLimits.SystemBaseLimit.ConnsInbound { // Conns should be at least 2x larger than the high water to allow for two conns per peer (TCP+QUIC). - // - // Outbound conns are set very high to allow for the accelerated DHT client to refresh its routing table. - // Currently it does not gracefully handle RM throttling, once it does we can lower this. defaultLimits.SystemBaseLimit.ConnsInbound = logScale(2 * maxconns) - defaultLimits.SystemBaseLimit.ConnsOutbound = logScale(64 * maxconns) - defaultLimits.SystemBaseLimit.Conns = defaultLimits.SystemBaseLimit.ConnsOutbound + defaultLimits.SystemBaseLimit.ConnsInbound - defaultLimits.SystemBaseLimit.StreamsInbound = defaultLimits.SystemBaseLimit.ConnsInbound * 4 - defaultLimits.SystemBaseLimit.StreamsOutbound = defaultLimits.SystemBaseLimit.ConnsOutbound * 16 - defaultLimits.SystemBaseLimit.Streams = defaultLimits.SystemBaseLimit.Conns * 16 + // We want the floor of outbound conns to be no less than what was set above. + if outbound := logScale(2 * maxconns); outbound > defaultLimits.SystemBaseLimit.ConnsOutbound { + defaultLimits.SystemBaseLimit.ConnsOutbound = outbound + } if 2*maxconns > defaultLimits.SystemBaseLimit.FD { defaultLimits.SystemBaseLimit.FD = logScale(2 * maxconns) } + defaultLimits.SystemBaseLimit.StreamsInbound = logScale(16 * maxconns) + defaultLimits.SystemBaseLimit.StreamsOutbound = logScale(64 * maxconns) + defaultLimits.SystemBaseLimit.Streams = logScale(64 * maxconns) + defaultLimits.ServiceBaseLimit.StreamsInbound = logScale(8 * maxconns) defaultLimits.ServiceBaseLimit.StreamsOutbound = logScale(32 * maxconns) defaultLimits.ServiceBaseLimit.Streams = logScale(32 * maxconns) @@ -64,6 +67,8 @@ func adjustedDefaultLimits(cfg config.SwarmConfig) rcmgr.DefaultLimitConfig { } } + defaultLimits.SystemBaseLimit.Conns = defaultLimits.SystemBaseLimit.ConnsOutbound + defaultLimits.SystemBaseLimit.ConnsInbound + return defaultLimits } From f26c96c9c8c3be9239b7c501816e76c2d5948928 Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Thu, 2 Jun 2022 10:05:09 -0400 Subject: [PATCH 3/3] increase min FD limit --- core/node/libp2p/rcmgr_defaults.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/core/node/libp2p/rcmgr_defaults.go b/core/node/libp2p/rcmgr_defaults.go index ddc6a82da2b..3f754fce433 100644 --- a/core/node/libp2p/rcmgr_defaults.go +++ b/core/node/libp2p/rcmgr_defaults.go @@ -31,11 +31,16 @@ func adjustedDefaultLimits(cfg config.SwarmConfig) rcmgr.DefaultLimitConfig { // - if Swarm.ConnMgr.HighWater is too high, adjust Conn/FD/Stream limits defaultLimits := rcmgr.DefaultLimits.WithSystemMemory(.125, 1<<30, 4<<30) - // Outbound conns are set very high to allow for the accelerated DHT client to (re)load its routing table. - // Currently it doesn't gracefully handle RM throttling--once it does we can lower this. + // Outbound conns and FDs are set very high to allow for the accelerated DHT client to (re)load its routing table. + // Currently it doesn't gracefully handle RM throttling--once it does we can lower these. // High outbound conn limits are considered less of a DoS risk than high inbound conn limits. // Also note that, due to the behavior of the accelerated DHT client, we don't need many streams, just conns. - defaultLimits.SystemBaseLimit.ConnsOutbound = 65536 + if minOutbound := 65536; defaultLimits.SystemBaseLimit.ConnsOutbound < minOutbound { + defaultLimits.SystemBaseLimit.ConnsOutbound = minOutbound + } + if minFD := 4096; defaultLimits.SystemBaseLimit.FD < minFD { + defaultLimits.SystemBaseLimit.FD = minFD + } // Do we need to adjust due to Swarm.ConnMgr.HighWater? if cfg.ConnMgr.Type == "basic" { @@ -44,9 +49,9 @@ func adjustedDefaultLimits(cfg config.SwarmConfig) rcmgr.DefaultLimitConfig { // Conns should be at least 2x larger than the high water to allow for two conns per peer (TCP+QUIC). defaultLimits.SystemBaseLimit.ConnsInbound = logScale(2 * maxconns) - // We want the floor of outbound conns to be no less than what was set above. - if outbound := logScale(2 * maxconns); outbound > defaultLimits.SystemBaseLimit.ConnsOutbound { - defaultLimits.SystemBaseLimit.ConnsOutbound = outbound + // We want the floor of minOutbound conns to be no less than what was set above. + if minOutbound := logScale(2 * maxconns); minOutbound > defaultLimits.SystemBaseLimit.ConnsOutbound { + defaultLimits.SystemBaseLimit.ConnsOutbound = minOutbound } if 2*maxconns > defaultLimits.SystemBaseLimit.FD {