Skip to content

Commit

Permalink
Merge pull request #591 from onflow/petera/add-profiler
Browse files Browse the repository at this point in the history
Add profiler server for pprof
  • Loading branch information
peterargue authored Sep 26, 2024
2 parents e7a47eb + 4487599 commit 1d311ad
Show file tree
Hide file tree
Showing 5 changed files with 173 additions and 35 deletions.
12 changes: 11 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,14 @@ start:
start-local:
rm -rf db/
rm -rf metrics/data/
go run cmd/main/main.go --flow-network-id=flow-emulator --coinbase=FACF71692421039876a5BB4F10EF7A439D8ef61E --coa-address=f8d6e0586b0a20c7 --coa-key=2619878f0e2ff438d17835c2a4561cb87b4d24d72d12ec34569acd0dd4af7c21 --wallet-api-key=2619878f0e2ff438d17835c2a4561cb87b4d24d72d12ec34569acd0dd4af7c21 --coa-resource-create=true --gas-price=0 --log-writer=console
go run cmd/main/main.go \
--flow-network-id=flow-emulator \
--coinbase=FACF71692421039876a5BB4F10EF7A439D8ef61E \
--coa-address=f8d6e0586b0a20c7 \
--coa-key=2619878f0e2ff438d17835c2a4561cb87b4d24d72d12ec34569acd0dd4af7c21 \
--wallet-api-key=2619878f0e2ff438d17835c2a4561cb87b4d24d72d12ec34569acd0dd4af7c21 \
--coa-resource-create=true \
--gas-price=0 \
--log-writer=console \
--profiler-enabled=true \
--profiler-port=6060
90 changes: 56 additions & 34 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -172,40 +172,43 @@ Running the EVM gateway for mainnet requires additional security and stability m

The application can be configured using the following flags at runtime:

| Flag | Default Value | Description |
|------------------------------|---------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `database-dir` | `./db` | Path to the directory for the database |
| `rpc-host` | `""` | Host for the RPC API server |
| `rpc-port` | `8545` | Port for the RPC API server |
| `ws-enabled` | `false` | Enable websocket connections |
| `access-node-grpc-host` | `localhost:3569` | Host to the flow access node gRPC API |
| `access-node-spork-hosts` | `""` | Previous spork AN hosts, defined as a comma-separated list (e.g. `"host-1.com,host2.com"`) |
| `evm-network-id` | `testnet ` | EVM network ID (options: `testnet`, `mainnet`) |
| `flow-network-id` | `flow-emulator` | Flow network ID (options: `flow-emulator`, `flow-testnet`, `flow-mainnet`) |
| `coinbase` | `""` | Coinbase address to use for fee collection |
| `init-cadence-height` | `0` | Cadence block height to start indexing; avoid using on a new network |
| `gas-price` | `1` | Static gas price for EVM transactions |
| `coa-address` | `""` | Flow address holding COA account for submitting transactions |
| `coa-key` | `""` | Private key for the COA address used for transactions |
| `coa-key-file` | `""` | Path to a JSON file of COA keys for key-rotation (exclusive with `coa-key` flag) |
| `coa-resource-create` | `false` | Auto-create the COA resource if it doesn't exist in the Flow COA account |
| `coa-cloud-kms-project-id` | `""` | Project ID for KMS keys (e.g. `flow-evm-gateway`) |
| `coa-cloud-kms-location-id` | `""` | Location ID for KMS key ring (e.g. 'global') |
| `coa-cloud-kms-key-ring-id` | `""` | Key ring ID for KMS keys (e.g. 'tx-signing') |
| `coa-cloud-kms-keys` | `""` | KMS keys and versions, comma-separated (e.g. `"gw-key-6@1,gw-key-7@1"`) |
| `log-level` | `debug` | Log verbosity level (`debug`, `info`, `warn`, `error`, `fatal`, `panic`) |
| `log-writer` | `stderr` | Output method for logs (`stderr`, `console`) |
| `stream-limit` | `10` | Rate-limit for client events sent per second |
| `rate-limit` | `50` | Requests per second limit for clients over any protocol (ws/http) |
| `address-header` | `""` | Header for client IP when server is behind a proxy |
| `heartbeat-interval` | `100` | Interval for AN event subscription heartbeats |
| `stream-timeout` | `3` | Timeout in seconds for sending events to clients |
| `force-start-height` | `0` | Force-set starting Cadence height (local/testing use only) |
| `wallet-api-key` | `""` | ECDSA private key for wallet APIs (local/testing use only) |
| `filter-expiry` | `5m` | Expiry time for idle filters |
| `traces-gcp-bucket` | `""` | GCP bucket name for transaction traces |
| `prometheus-config-file-path`| `./metrics/prometheus.yml` | Path to the Prometheus configuration file |
| `index-only` | `false` | Run in index-only mode, allowing state queries and indexing but no transaction sending |
| Flag | Default Value | Description |
|------------------------------|---------------------------------|------------------------------------------------------------------------------------------|
| `database-dir` | `./db` | Path to the directory for the database |
| `rpc-host` | `""` | Host for the RPC API server |
| `rpc-port` | `8545` | Port for the RPC API server |
| `ws-enabled` | `false` | Enable websocket connections |
| `access-node-grpc-host` | `localhost:3569` | Host to the flow access node gRPC API |
| `access-node-spork-hosts` | `""` | Previous spork AN hosts, defined as a comma-separated list (e.g. `"host-1.com,host2.com"`) |
| `evm-network-id` | `testnet ` | EVM network ID (options: `testnet`, `mainnet`) |
| `flow-network-id` | `flow-emulator` | Flow network ID (options: `flow-emulator`, `flow-testnet`, `flow-mainnet`) |
| `coinbase` | `""` | Coinbase address to use for fee collection |
| `init-cadence-height` | `0` | Cadence block height to start indexing; avoid using on a new network |
| `gas-price` | `1` | Static gas price for EVM transactions |
| `coa-address` | `""` | Flow address holding COA account for submitting transactions |
| `coa-key` | `""` | Private key for the COA address used for transactions |
| `coa-key-file` | `""` | Path to a JSON file of COA keys for key-rotation (exclusive with `coa-key` flag) |
| `coa-resource-create` | `false` | Auto-create the COA resource if it doesn't exist in the Flow COA account |
| `coa-cloud-kms-project-id` | `""` | Project ID for KMS keys (e.g. `flow-evm-gateway`) |
| `coa-cloud-kms-location-id` | `""` | Location ID for KMS key ring (e.g. 'global') |
| `coa-cloud-kms-key-ring-id` | `""` | Key ring ID for KMS keys (e.g. 'tx-signing') |
| `coa-cloud-kms-keys` | `""` | KMS keys and versions, comma-separated (e.g. `"gw-key-6@1,gw-key-7@1"`) |
| `log-level` | `debug` | Log verbosity level (`debug`, `info`, `warn`, `error`, `fatal`, `panic`) |
| `log-writer` | `stderr` | Output method for logs (`stderr`, `console`) |
| `stream-limit` | `10` | Rate-limit for client events sent per second |
| `rate-limit` | `50` | Requests per second limit for clients over any protocol (ws/http) |
| `address-header` | `""` | Header for client IP when server is behind a proxy |
| `heartbeat-interval` | `100` | Interval for AN event subscription heartbeats |
| `stream-timeout` | `3` | Timeout in seconds for sending events to clients |
| `force-start-height` | `0` | Force-set starting Cadence height (local/testing use only) |
| `wallet-api-key` | `""` | ECDSA private key for wallet APIs (local/testing use only) |
| `filter-expiry` | `5m` | Expiry time for idle filters |
| `traces-gcp-bucket` | `""` | GCP bucket name for transaction traces |
| `prometheus-config-file-path`| `./metrics/prometheus.yml` | Path to the Prometheus configuration file |
| `index-only` | `false` | Run in index-only mode, allowing state queries and indexing but no transaction sending |
| `profiler-enabled` | `false` | Enable the pprof profiler server |
| `profiler-host` | `localhost` | Host for the pprof profiler |
| `profiler-port` | `6060` | Port for the pprof profiler |

# Deploying
Deploying the EVM Gateway node comes with some prerequisites as well as expectations and they are best explained in the WIP document: https://flowfoundation.notion.site/EVM-Gateway-Deployment-3c41da6710af40acbaf971e22ce0a9fd
Expand Down Expand Up @@ -252,6 +255,25 @@ The EVM Gateway implements APIs according to the Ethereum specification: https:/
- Access Lists: we don't yet support creating access lists as they don't affect the fees we charge. We might support this in the future
to optimize fees, but it currently is not part of our priorities.

# Debugging

## Profiler

The EVM Gateway supports profiling via the `pprof` package. To enable profiling, add the following flags to the command line:
```
--profiler-enabled=true
--profiler-host=localhost
--profiler-port=6060
```

This will start a pprof server on the provided `host` and `port`. You can generate profiles using the following `go tool` commands
```
go tool pprof -http :2000 http://localhost:6060/debug/pprof/profile
```
```
curl --output trace.out http://localhost:6060/debug/pprof/trace
go tool trace -http :2001 trace.out
```

# Contributing
We welcome contributions from the community! Please read our [Contributing Guide](./CONTRIBUTING.md) for information on how to get involved.
Expand Down
60 changes: 60 additions & 0 deletions api/profiler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package api

import (
"context"
"errors"
"net"
"net/http"
_ "net/http/pprof"
"strconv"

"github.com/rs/zerolog"
)

type ProfileServer struct {
logger zerolog.Logger
server *http.Server
endpoint string
}

func NewProfileServer(
logger zerolog.Logger,
host string,
port int,
) *ProfileServer {
endpoint := net.JoinHostPort(host, strconv.Itoa(port))
return &ProfileServer{
logger: logger,
server: &http.Server{Addr: endpoint},
endpoint: endpoint,
}
}

func (h *ProfileServer) ListenAddr() string {
return h.endpoint
}

func (s *ProfileServer) Start() {
go func() {
err := s.server.ListenAndServe()
if err != nil {
if errors.Is(err, http.ErrServerClosed) {
s.logger.Warn().Msg("Profiler server shutdown")
return
}
s.logger.Err(err).Msg("failed to start Profiler server")
panic(err)
}
}()
}

func (s *ProfileServer) Stop() error {
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()

return s.server.Shutdown(ctx)
}

func (s *ProfileServer) Close() error {
return s.server.Close()
}
37 changes: 37 additions & 0 deletions bootstrap/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ type Bootstrap struct {
metrics *metrics.Server
events *ingestion.Engine
traces *traces.Engine
profiler *api.ProfileServer
}

func New(config *config.Config) (*Bootstrap, error) {
Expand Down Expand Up @@ -340,6 +341,38 @@ func (b *Bootstrap) StopMetricsServer() {
b.metrics.Stop()
}

func (b *Bootstrap) StartProfilerServer(_ context.Context) error {
if !b.config.ProfilerEnabled {
return nil
}
b.logger.Info().Msg("bootstrap starting profiler server")

b.profiler = api.NewProfileServer(b.logger, b.config.ProfilerHost, b.config.ProfilerPort)

b.profiler.Start()
b.logger.Info().Msgf("Profiler server started: %s", b.profiler.ListenAddr())

return nil
}

func (b *Bootstrap) StopProfilerServer() {
if b.profiler == nil {
return
}

b.logger.Warn().Msg("shutting down profiler server")

err := b.profiler.Stop()
if err != nil {
if errors.Is(err, context.DeadlineExceeded) {
b.logger.Warn().Msg("Profiler server graceful shutdown timed out")
b.profiler.Close()
} else {
b.logger.Err(err).Msg("Profiler server graceful shutdown failed")
}
}
}

// startEngine starts provided engine and panics if there are startup errors.
func startEngine(
ctx context.Context,
Expand Down Expand Up @@ -478,6 +511,10 @@ func Run(ctx context.Context, cfg *config.Config, ready chan struct{}) error {
return fmt.Errorf("failed to start metrics server: %w", err)
}

if err := boot.StartProfilerServer(ctx); err != nil {
return fmt.Errorf("failed to start profiler server: %w", err)
}

// mark ready
close(ready)

Expand Down
9 changes: 9 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ type Config struct {
IndexOnly bool
// Cache size in units of items in cache, one unit in cache takes approximately 64 bytes
CacheSize uint
// ProfilerEnabled sets whether the profiler server is enabled
ProfilerEnabled bool
// ProfilerHost is the host for the profiler server will listen to (e.g. localhost, 0.0.0.0)
ProfilerHost string
// ProfilerPort is the port for the profiler server
ProfilerPort int
}

func FromFlags() (*Config, error) {
Expand Down Expand Up @@ -159,6 +165,9 @@ func FromFlags() (*Config, error) {
flag.StringVar(&walletKey, "wallet-api-key", "", "ECDSA private key used for wallet APIs. WARNING: This should only be used locally or for testing, never in production.")
flag.IntVar(&cfg.MetricsPort, "metrics-port", 9091, "Port for the metrics server")
flag.BoolVar(&cfg.IndexOnly, "index-only", false, "Run the gateway in index-only mode which only allows querying the state and indexing, but disallows sending transactions.")
flag.BoolVar(&cfg.ProfilerEnabled, "profiler-enabled", false, "Run the profiler server to capture pprof data.")
flag.StringVar(&cfg.ProfilerHost, "profiler-host", "localhost", "Host for the Profiler server")
flag.IntVar(&cfg.ProfilerPort, "profiler-port", 6060, "Port for the Profiler server")
flag.Parse()

if coinbase == "" {
Expand Down

0 comments on commit 1d311ad

Please sign in to comment.