From a60b80e8aa4286dc770df2af05fc6173f28348d5 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 10 Apr 2024 12:36:10 -0500 Subject: [PATCH 1/3] fix: alert: fix up alerts with respect to fvm concurrency High values are now usually OK because we "oversubscribe" fvm instances. The old warning of 24 is now equivalent to 1k+. --- node/modules/alerts.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/node/modules/alerts.go b/node/modules/alerts.go index e0aa0977a85..a3c0e20d292 100644 --- a/node/modules/alerts.go +++ b/node/modules/alerts.go @@ -117,13 +117,20 @@ func CheckFvmConcurrency() func(al *alerting.Alerting) { return } - // Raise alert if LOTUS_FVM_CONCURRENCY is set to a high value - if fvmConcurrencyVal > 24 { + if fvmConcurrencyVal < 2 { alert := al.AddAlertType("process", "fvm-concurrency") al.Raise(alert, map[string]interface{}{ - "message": "LOTUS_FVM_CONCURRENCY is set to a high value that can cause chain sync panics on network migrations/upgrades", + "message": "LOTUS_FVM_CONCURRENCY should be at least 2", "set_value": fvmConcurrencyVal, - "recommended": "24 or less during network upgrades", + "recommended": "set it to two or more", + }) + } + if fvmConcurrencyVal > 512 { + alert := al.AddAlertType("process", "fvm-concurrency") + al.Raise(alert, map[string]interface{}{ + "message": "LOTUS_FVM_CONCURRENCY is set to a high value that can cause chain sync panics on network migrations/upgrades and excessive memory usage (>32GiB)", + "set_value": fvmConcurrencyVal, + "recommended": "leave it unset to default to the number of available cpu threads", }) } } From 9d8ec0a428ae3f899473dc7a5beaf5b71aa0fb7e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 14 Apr 2024 10:47:47 -0500 Subject: [PATCH 2/3] fix: fvm: set default max lanes to num CPUs --- chain/vm/execution.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/chain/vm/execution.go b/chain/vm/execution.go index 4fb626f4390..ae56422dacd 100644 --- a/chain/vm/execution.go +++ b/chain/vm/execution.go @@ -3,6 +3,7 @@ package vm import ( "context" "os" + "runtime" "strconv" "sync" @@ -14,16 +15,15 @@ import ( "github.com/filecoin-project/lotus/metrics" ) -const ( - // DefaultAvailableExecutionLanes is the number of available execution lanes; it is the bound of - // concurrent active executions. - // This is the default value in filecoin-ffi - DefaultAvailableExecutionLanes = 4 - // DefaultPriorityExecutionLanes is the number of reserved execution lanes for priority computations. - // This is purely userspace, but we believe it is a reasonable default, even with more available - // lanes. - DefaultPriorityExecutionLanes = 2 -) +// DefaultPriorityExecutionLanes is the number of reserved execution lanes for priority computations. +// This is purely userspace, but we believe it is a reasonable default, even with more available +// lanes. +const DefaultPriorityExecutionLanes = 2 + +// DefaultAvailableExecutionLanes is the number of available execution lanes; it is the bound of +// concurrent active executions. +// This is the default value in filecoin-ffi +var DefaultAvailableExecutionLanes = runtime.NumCPU() // the execution environment; see below for definition, methods, and initialization var execution *executionEnv From 55a71c4c1a45643ce4d5181352f7b799e244efd7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 19 Jun 2024 09:49:45 -0700 Subject: [PATCH 3/3] chore: update changelog to record FVM concurrency changes --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95ad61a383e..5add9315c1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,16 @@ There is no change in the behaviour when a call returns an error, as the error o ## New features +### FVM RPC Concurrent Requests + +The default maximum FVM concurrency (`LOTUS_FVM_CONCURRENCY`) has increased from 4 to the number of available CPU threads. Furthermore, the FVM now allocates `LOTUS_FVM_CONCURRENCY` OS-level threads with 64MiB stacks (in 1.26, we allocated one thread per CPU core regardless of the value of `LOTUS_FVM_CONCURRENCY`). + +Impact: + +1. This will increase memory usage for nodes specifying a higher `LOTUS_FVM_CONCURRENCY` than the number of CPU threads they have available. +2. This will allow such nodes to actually take advantage of this higher concurrency limit where the concurrency was previously capped at the number of available CPU threads. + + ## Improvements # v1.27.0 / 2024-05-27