From 180442181eb219a7724dba23657e7dbae665045c Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 10:42:25 +0200 Subject: [PATCH 01/12] Start grpc server and register services in standalone mode --- server/start.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/server/start.go b/server/start.go index 5e4383b1422e..9bab294ab95d 100644 --- a/server/start.go +++ b/server/start.go @@ -258,12 +258,12 @@ func start(svrCtx *Context, clientCtx client.Context, appCreator types.AppCreato emitServerInfoMetrics() if !withCmt { - return startStandAlone(svrCtx, app, opts) + return startStandAlone(svrCtx, svrCfg, clientCtx, app, opts) } return startInProcess(svrCtx, svrCfg, clientCtx, app, metrics, opts) } -func startStandAlone(svrCtx *Context, app types.Application, opts StartCmdOptions) error { +func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx client.Context, app types.Application, opts StartCmdOptions) error { addr := svrCtx.Viper.GetString(flagAddress) transport := svrCtx.Viper.GetString(flagTransport) @@ -277,6 +277,19 @@ func startStandAlone(svrCtx *Context, app types.Application, opts StartCmdOption g, ctx := getCtx(svrCtx, false) + _, clientCtx, err = startGrpcServer(ctx, g, svrCfg.GRPC, clientCtx, svrCtx, app) + if err != nil { + return err + } + + // Add the tx service to the gRPC router. + if svrCfg.GRPC.Enable { + // use the provided clientCtx to register the services + app.RegisterTxService(clientCtx) + app.RegisterTendermintService(clientCtx) + app.RegisterNodeService(clientCtx, svrCfg) + } + g.Go(func() error { if err := svr.Start(); err != nil { svrCtx.Logger.Error("failed to start out-of-process ABCI server", "err", err) From 7a7a1a82f4b6aa44ef8deda8726edabc7b47b248 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 13:49:04 +0200 Subject: [PATCH 02/12] Use RPC client for standalone mode --- server/start.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/start.go b/server/start.go index 9bab294ab95d..ffbc90b1f35f 100644 --- a/server/start.go +++ b/server/start.go @@ -26,6 +26,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + rpchttp "github.com/cometbft/cometbft/rpc/client/http" + pruningtypes "cosmossdk.io/store/pruning/types" "github.com/cosmos/cosmos-sdk/client" @@ -284,6 +286,16 @@ func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx clie // Add the tx service to the gRPC router. if svrCfg.GRPC.Enable { + // create tendermint client + // assumes the rpc listen address is where tendermint has its rpc server + rpcclient, err := rpchttp.New(svrCtx.Config.RPC.ListenAddress, "/websocket") + if err != nil { + return err + } + // re-assign for making the client available below + // do not use := to avoid shadowing clientCtx + clientCtx = clientCtx.WithClient(rpcclient) + // use the provided clientCtx to register the services app.RegisterTxService(clientCtx) app.RegisterTendermintService(clientCtx) From 3cbf2e62d61e80f1718283197c9a1f2e61e0035e Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 14:31:42 +0200 Subject: [PATCH 03/12] Register services before starting grpc server --- server/start.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/start.go b/server/start.go index ffbc90b1f35f..1ec0b716d33a 100644 --- a/server/start.go +++ b/server/start.go @@ -279,11 +279,6 @@ func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx clie g, ctx := getCtx(svrCtx, false) - _, clientCtx, err = startGrpcServer(ctx, g, svrCfg.GRPC, clientCtx, svrCtx, app) - if err != nil { - return err - } - // Add the tx service to the gRPC router. if svrCfg.GRPC.Enable { // create tendermint client @@ -302,6 +297,11 @@ func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx clie app.RegisterNodeService(clientCtx, svrCfg) } + _, clientCtx, err = startGrpcServer(ctx, g, svrCfg.GRPC, clientCtx, svrCtx, app) + if err != nil { + return err + } + g.Go(func() error { if err := svr.Start(); err != nil { svrCtx.Logger.Error("failed to start out-of-process ABCI server", "err", err) From 8f7c8169df23c258b531fbcea17ecb7f1d9f4450 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 14:31:58 +0200 Subject: [PATCH 04/12] Remove unused reference to clientCtx --- server/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/start.go b/server/start.go index 1ec0b716d33a..7834c7b33595 100644 --- a/server/start.go +++ b/server/start.go @@ -297,7 +297,7 @@ func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx clie app.RegisterNodeService(clientCtx, svrCfg) } - _, clientCtx, err = startGrpcServer(ctx, g, svrCfg.GRPC, clientCtx, svrCtx, app) + _, _, err = startGrpcServer(ctx, g, svrCfg.GRPC, clientCtx, svrCtx, app) if err != nil { return err } From 4c21fd55df56323a85e6e835d97a1be5092d01ef Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 14:48:00 +0200 Subject: [PATCH 05/12] Fix import order --- server/start.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/start.go b/server/start.go index 7834c7b33595..f69cf05738b3 100644 --- a/server/start.go +++ b/server/start.go @@ -16,6 +16,7 @@ import ( "github.com/cometbft/cometbft/p2p" pvm "github.com/cometbft/cometbft/privval" "github.com/cometbft/cometbft/proxy" + rpchttp "github.com/cometbft/cometbft/rpc/client/http" "github.com/cometbft/cometbft/rpc/client/local" cmttypes "github.com/cometbft/cometbft/types" dbm "github.com/cosmos/cosmos-db" @@ -26,8 +27,6 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - rpchttp "github.com/cometbft/cometbft/rpc/client/http" - pruningtypes "cosmossdk.io/store/pruning/types" "github.com/cosmos/cosmos-sdk/client" From 7d9004130901397819b14e3aa29b1d7e22ad4221 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 14:50:01 +0200 Subject: [PATCH 06/12] Add entry to CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b01825915d7..31e26683f753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Features +* (server) [#18110](https://github.com/cosmos/cosmos-sdk/pull/18110) Start gRPC server in standalone mode * (tests) [#17868](https://github.com/cosmos/cosmos-sdk/pull/17868) Added helper method `SubmitTestTx` in testutil to broadcast test txns to test e2e tests. * (x/protocolpool) [#17657](https://github.com/cosmos/cosmos-sdk/pull/17657) Create a new `x/protocolpool` module that is responsible for handling community pool funds. This module is split out into a new module from x/distribution. * (baseapp) [#16581](https://github.com/cosmos/cosmos-sdk/pull/16581) Implement Optimistic Execution as an experimental feature (not enabled by default). From 88dd555e94628d51562d29e10eb5f390f75d42c2 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 14:52:36 +0200 Subject: [PATCH 07/12] Use comment from inProcess mode for more uniformity --- server/start.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/start.go b/server/start.go index f69cf05738b3..fb1cc7490899 100644 --- a/server/start.go +++ b/server/start.go @@ -278,7 +278,8 @@ func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx clie g, ctx := getCtx(svrCtx, false) - // Add the tx service to the gRPC router. + // Add the tx service to the gRPC router. We only need to register this + // service if gRPC is enabled. if svrCfg.GRPC.Enable { // create tendermint client // assumes the rpc listen address is where tendermint has its rpc server From 3b466bd32d1731775e891e8b84e292a7607ae4c1 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 16:16:29 +0200 Subject: [PATCH 08/12] Add --node flag to startup to support standalone comet --- server/start.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/start.go b/server/start.go index fb1cc7490899..7aafe3a73ba6 100644 --- a/server/start.go +++ b/server/start.go @@ -89,6 +89,9 @@ const ( // mempool flags FlagMempoolMaxTxs = "mempool.max-txs" + + // standalone node flags + FlagNode = "node" ) // StartCmdOptions defines options that can be customized in `StartCmdWithOptions`, @@ -220,6 +223,7 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. cmd.Flags().Bool(FlagDisableIAVLFastNode, false, "Disable fast node for IAVL tree") cmd.Flags().Int(FlagMempoolMaxTxs, mempool.DefaultMaxTx, "Sets MaxTx value for the app-side mempool") cmd.Flags().Duration(FlagShutdownGrace, 0*time.Second, "On Shutdown, duration to wait for resource clean up") + cmd.Flags().String(FlagNode, "tcp://localhost:26657", ": to CometBFT RPC interface for this chain") // support old flags name for backwards compatibility cmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { @@ -283,7 +287,7 @@ func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx clie if svrCfg.GRPC.Enable { // create tendermint client // assumes the rpc listen address is where tendermint has its rpc server - rpcclient, err := rpchttp.New(svrCtx.Config.RPC.ListenAddress, "/websocket") + rpcclient, err := rpchttp.New(clientCtx.NodeURI, "/websocket") if err != nil { return err } From d3699060f65b1c4bcc93738cd98db4ef4d16d989 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 16:44:21 +0200 Subject: [PATCH 09/12] Revert "Add --node flag to startup to support standalone comet" This reverts commit 3b466bd32d1731775e891e8b84e292a7607ae4c1. --- server/start.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/start.go b/server/start.go index 7aafe3a73ba6..fb1cc7490899 100644 --- a/server/start.go +++ b/server/start.go @@ -89,9 +89,6 @@ const ( // mempool flags FlagMempoolMaxTxs = "mempool.max-txs" - - // standalone node flags - FlagNode = "node" ) // StartCmdOptions defines options that can be customized in `StartCmdWithOptions`, @@ -223,7 +220,6 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. cmd.Flags().Bool(FlagDisableIAVLFastNode, false, "Disable fast node for IAVL tree") cmd.Flags().Int(FlagMempoolMaxTxs, mempool.DefaultMaxTx, "Sets MaxTx value for the app-side mempool") cmd.Flags().Duration(FlagShutdownGrace, 0*time.Second, "On Shutdown, duration to wait for resource clean up") - cmd.Flags().String(FlagNode, "tcp://localhost:26657", ": to CometBFT RPC interface for this chain") // support old flags name for backwards compatibility cmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { @@ -287,7 +283,7 @@ func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx clie if svrCfg.GRPC.Enable { // create tendermint client // assumes the rpc listen address is where tendermint has its rpc server - rpcclient, err := rpchttp.New(clientCtx.NodeURI, "/websocket") + rpcclient, err := rpchttp.New(svrCtx.Config.RPC.ListenAddress, "/websocket") if err != nil { return err } From 9ed0aa7184d6d251e9f243a01e5253cf72ec1f32 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Fri, 13 Oct 2023 16:50:57 +0200 Subject: [PATCH 10/12] Add paragraph about standalone CometBFT to README --- docs/user/run-node/01-run-node.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/user/run-node/01-run-node.md b/docs/user/run-node/01-run-node.md index a9ef422b6889..f16eb42f52de 100644 --- a/docs/user/run-node/01-run-node.md +++ b/docs/user/run-node/01-run-node.md @@ -157,6 +157,13 @@ The previous command allow you to run a single node. This is enough for the next The naive way would be to run the same commands again in separate terminal windows. This is possible, however in the Cosmos SDK, we leverage the power of [Docker Compose](https://docs.docker.com/compose/) to run a localnet. If you need inspiration on how to set up your own localnet with Docker Compose, you can have a look at the Cosmos SDK's [`docker-compose.yml`](https://github.com/cosmos/cosmos-sdk/blob/v0.50.0-alpha.0/docker-compose.yml). +### Standalone App/CometBFT + +By default, the Cosmos SDK runs CometBFT in-process with the application +If you want to run the application and CometBFT in separate processes, +start the application with the `--with-comet=false` flag +and set `rpc.laddr` in `config.toml` to the CometBFT node's RPC address. + ## Logging Logging provides a way to see what is going on with a node. By default the info level is set. This is a global level and all info logs will be outputted to the terminal. If you would like to filter specific logs to the terminal instead of all, then setting `module:log_level` is how this can work. From 0cf2b36bc667ab69f2e31dd2477e110baf9dac47 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt Date: Mon, 16 Oct 2023 14:24:24 +0200 Subject: [PATCH 11/12] Start API server in standalone mode --- server/start.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/server/start.go b/server/start.go index fb1cc7490899..008fb2e1864e 100644 --- a/server/start.go +++ b/server/start.go @@ -259,12 +259,12 @@ func start(svrCtx *Context, clientCtx client.Context, appCreator types.AppCreato emitServerInfoMetrics() if !withCmt { - return startStandAlone(svrCtx, svrCfg, clientCtx, app, opts) + return startStandAlone(svrCtx, svrCfg, clientCtx, app, metrics, opts) } return startInProcess(svrCtx, svrCfg, clientCtx, app, metrics, opts) } -func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx client.Context, app types.Application, opts StartCmdOptions) error { +func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx client.Context, app types.Application, metrics *telemetry.Metrics, opts StartCmdOptions) error { addr := svrCtx.Viper.GetString(flagAddress) transport := svrCtx.Viper.GetString(flagTransport) @@ -279,8 +279,9 @@ func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx clie g, ctx := getCtx(svrCtx, false) // Add the tx service to the gRPC router. We only need to register this - // service if gRPC is enabled. - if svrCfg.GRPC.Enable { + // service if API or gRPC is enabled, and avoid doing so in the general + // case, because it spawns a new local CometBFT RPC client. + if svrCfg.API.Enable || svrCfg.GRPC.Enable { // create tendermint client // assumes the rpc listen address is where tendermint has its rpc server rpcclient, err := rpchttp.New(svrCtx.Config.RPC.ListenAddress, "/websocket") @@ -297,7 +298,15 @@ func startStandAlone(svrCtx *Context, svrCfg serverconfig.Config, clientCtx clie app.RegisterNodeService(clientCtx, svrCfg) } - _, _, err = startGrpcServer(ctx, g, svrCfg.GRPC, clientCtx, svrCtx, app) + grpcSrv, clientCtx, err := startGrpcServer(ctx, g, svrCfg.GRPC, clientCtx, svrCtx, app) + if err != nil { + return err + } + + cmtCfg := svrCtx.Config + home := cmtCfg.RootDir + + err = startAPIServer(ctx, g, cmtCfg, svrCfg, clientCtx, svrCtx, app, home, grpcSrv, metrics) if err != nil { return err } From 0e35640f2d2dcd18b3bd4d2cb9a52cfc5ff4429c Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 18 Oct 2023 12:21:00 +0200 Subject: [PATCH 12/12] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b347596f36e..97bf73f19f6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Features -* (server) [#18110](https://github.com/cosmos/cosmos-sdk/pull/18110) Start gRPC server in standalone mode +* (server) [#18110](https://github.com/cosmos/cosmos-sdk/pull/18110) Start gRPC & API server in standalone mode * (client) [#18101](https://github.com/cosmos/cosmos-sdk/pull/18101) Add a `keyring-default-keyname` in `client.toml` for specifying a default key name, and skip the need to use the `--from` flag when signing transactions. * (tests) [#17868](https://github.com/cosmos/cosmos-sdk/pull/17868) Added helper method `SubmitTestTx` in testutil to broadcast test txns to test e2e tests. * (x/protocolpool) [#17657](https://github.com/cosmos/cosmos-sdk/pull/17657) Create a new `x/protocolpool` module that is responsible for handling community pool funds. This module is split out into a new module from x/distribution.