From b771effc6c755d553ffefc39f4e6bae8aa082a79 Mon Sep 17 00:00:00 2001 From: MD Aleem <72057206+aleem1314@users.noreply.github.com> Date: Mon, 18 Jan 2021 23:55:22 +0530 Subject: [PATCH] gRPC-web proxy (#8077) * init * WIP config * WIP add proxy server * fmt * WIP * setup proxy server * clean go.mod * lint * lint * lint * custom codec * lint * add comments * change grpc-proxy port * add grpc-web * update server/start.go * add tests * add test with client * Update server/start.go Co-authored-by: Anil Kumar Kammari * Update server/start.go Co-authored-by: Amaury * Update server/start.go Co-authored-by: Amaury * review changes * review changes * Update server/start.go Co-authored-by: Anil Kumar Kammari Co-authored-by: Amaury Co-authored-by: Aleksandr Bezobchuk Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- go.mod | 2 + go.sum | 9 + server/config/config.go | 23 ++- server/config/toml.go | 13 ++ server/grpc/grpc_web.go | 25 +++ server/grpc/grpc_web_test.go | 313 +++++++++++++++++++++++++++++++++++ server/start.go | 25 ++- testutil/network/network.go | 17 +- testutil/network/util.go | 18 ++ 9 files changed, 438 insertions(+), 7 deletions(-) create mode 100644 server/grpc/grpc_web.go create mode 100644 server/grpc/grpc_web_test.go diff --git a/go.mod b/go.mod index 10577caa095a..c507219f9663 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/cosmos/go-bip39 v1.0.0 github.com/cosmos/iavl v0.15.3 github.com/cosmos/ledger-cosmos-go v0.11.1 + github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/enigmampc/btcutil v1.0.3-0.20200723161021-e2fb6adb2a25 @@ -26,6 +27,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/golang-lru v0.5.4 + github.com/improbable-eng/grpc-web v0.13.0 github.com/magiconair/properties v1.8.4 github.com/mattn/go-isatty v0.0.12 github.com/otiai10/copy v1.4.2 diff --git a/go.sum b/go.sum index 30275e7ac493..aeab76d9c9e3 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= +github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgraph-io/badger/v2 v2.2007.1/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= @@ -253,6 +255,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-middleware v1.2.1/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 h1:FlFbCRLd5Jr4iYXZufAvgWN6Ao0JrI5chLINnUXDDr0= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -299,6 +302,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/improbable-eng/grpc-web v0.13.0 h1:7XqtaBWaOCH0cVGKHyvhtcuo6fgW32Y10yRKrDHFHOc= +github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= @@ -308,6 +313,7 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -327,6 +333,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kkdai/bstream v1.0.0/go.mod h1:FDnDOHt5Yx4p3FaHcioFT0QjDOtgUpvjeZqAs+NVZZA= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -373,6 +380,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= @@ -500,6 +508,7 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= diff --git a/server/config/config.go b/server/config/config.go index 80e79f04558d..68f6cf4352f6 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -14,8 +14,11 @@ import ( const ( defaultMinGasPrices = "" - // DefaultGRPCAddress is the default address the gRPC server binds to. + // DefaultGRPCAddress defines the default address to bind the gRPC server to. DefaultGRPCAddress = "0.0.0.0:9090" + + // DefaultGRPCWebAddress defines the default address to bind the gRPC-web server to. + DefaultGRPCWebAddress = "0.0.0.0:9091" ) // BaseConfig defines the server's basic configuration @@ -107,6 +110,15 @@ type GRPCConfig struct { Address string `mapstructure:"address"` } +// GRPCWebConfig defines configuration for the gRPC-web server. +type GRPCWebConfig struct { + // Enable defines if the gRPC-web should be enabled. + Enable bool `mapstructure:"enable"` + + // Address defines the gRPC-web server to listen on + Address string `mapstructure:"address"` +} + // StateSyncConfig defines the state sync snapshot configuration. type StateSyncConfig struct { // SnapshotInterval sets the interval at which state sync snapshots are taken. @@ -126,6 +138,7 @@ type Config struct { Telemetry telemetry.Config `mapstructure:"telemetry"` API APIConfig `mapstructure:"api"` GRPC GRPCConfig `mapstructure:"grpc"` + GRPCWeb GRPCWebConfig `mapstructure:"grpc-web"` StateSync StateSyncConfig `mapstructure:"state-sync"` } @@ -185,6 +198,10 @@ func DefaultConfig() *Config { Enable: true, Address: DefaultGRPCAddress, }, + GRPCWeb: GRPCWebConfig{ + Enable: true, + Address: DefaultGRPCWebAddress, + }, StateSync: StateSyncConfig{ SnapshotInterval: 0, SnapshotKeepRecent: 2, @@ -239,6 +256,10 @@ func GetConfig(v *viper.Viper) Config { Enable: v.GetBool("grpc.enable"), Address: v.GetString("grpc.address"), }, + GRPCWeb: GRPCWebConfig{ + Enable: v.GetBool("grpc-web.enable"), + Address: v.GetString("grpc-web.address"), + }, StateSync: StateSyncConfig{ SnapshotInterval: v.GetUint64("state-sync.snapshot-interval"), SnapshotKeepRecent: v.GetUint32("state-sync.snapshot-keep-recent"), diff --git a/server/config/toml.go b/server/config/toml.go index b042da74293d..58dc7bfed2b4 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -147,6 +147,19 @@ enable = {{ .GRPC.Enable }} # Address defines the gRPC server address to bind to. address = "{{ .GRPC.Address }}" +############################################################################### +### gRPC Web Configuration ### +############################################################################### + +[grpc-web] + +# GRPCWebEnable defines if the gRPC-web should be enabled. +# NOTE: gRPC must also be enabled, otherwise, this configuration is a no-op. +enable = {{ .GRPCWeb.Enable }} + +# Address defines the gRPC-web server address to bind to. +address = "{{ .GRPCWeb.Address }}" + ############################################################################### ### State Sync Configuration ### ############################################################################### diff --git a/server/grpc/grpc_web.go b/server/grpc/grpc_web.go new file mode 100644 index 000000000000..30c71808e43e --- /dev/null +++ b/server/grpc/grpc_web.go @@ -0,0 +1,25 @@ +package grpc + +import ( + "net/http" + + "github.com/cosmos/cosmos-sdk/server/config" + "github.com/improbable-eng/grpc-web/go/grpcweb" + "google.golang.org/grpc" +) + +// StartGRPCWeb starts a gRPC-Web server on the given address. +func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, error) { + wrappedServer := grpcweb.WrapServer(grpcSrv) + handler := func(resp http.ResponseWriter, req *http.Request) { + wrappedServer.ServeHTTP(resp, req) + } + grpcWebSrv := &http.Server{ + Addr: config.GRPCWeb.Address, + Handler: http.HandlerFunc(handler), + } + if err := grpcWebSrv.ListenAndServe(); err != nil { + return nil, err + } + return grpcWebSrv, nil +} diff --git a/server/grpc/grpc_web_test.go b/server/grpc/grpc_web_test.go new file mode 100644 index 000000000000..8f563becc666 --- /dev/null +++ b/server/grpc/grpc_web_test.go @@ -0,0 +1,313 @@ +package grpc_test + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/textproto" + "strconv" + "strings" + "testing" + + "github.com/golang/protobuf/proto" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc/codes" + + "github.com/cosmos/cosmos-sdk/client/grpc/tmservice" + "github.com/cosmos/cosmos-sdk/codec" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" + "github.com/cosmos/cosmos-sdk/testutil/network" + banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" +) + +// https://github.com/improbable-eng/grpc-web/blob/master/go/grpcweb/wrapper_test.go used as a reference +// to setup grpcRequest config. + +const grpcWebContentType = "application/grpc-web" + +type GRPCWebTestSuite struct { + suite.Suite + + cfg network.Config + network *network.Network + protoCdc *codec.ProtoCodec +} + +func (s *GRPCWebTestSuite) SetupSuite() { + s.T().Log("setting up integration test suite") + + cfg := network.DefaultConfig() + cfg.NumValidators = 1 + s.cfg = cfg + s.network = network.New(s.T(), s.cfg) + s.Require().NotNil(s.network) + + _, err := s.network.WaitForHeight(2) + s.Require().NoError(err) + + s.protoCdc = codec.NewProtoCodec(s.cfg.InterfaceRegistry) +} + +func (s *GRPCWebTestSuite) TearDownSuite() { + s.T().Log("tearing down integration test suite") + s.network.Cleanup() +} + +func (s *GRPCWebTestSuite) Test_Latest_Validators() { + val := s.network.Validators[0] + for _, contentType := range []string{grpcWebContentType} { + headers, trailers, responses, err := s.makeGrpcRequest( + "/cosmos.base.tendermint.v1beta1.Service/GetLatestValidatorSet", + headerWithFlag(), + serializeProtoMessages([]proto.Message{&tmservice.GetLatestValidatorSetRequest{}}), false) + + s.Require().NoError(err) + s.Require().Equal(1, len(responses)) + s.assertTrailerGrpcCode(trailers, codes.OK, "") + s.assertContentTypeSet(headers, contentType) + var valsSet tmservice.GetLatestValidatorSetResponse + err = s.protoCdc.UnmarshalBinaryBare(responses[0], &valsSet) + s.Require().NoError(err) + pubKey, ok := valsSet.Validators[0].PubKey.GetCachedValue().(cryptotypes.PubKey) + s.Require().Equal(true, ok) + s.Require().Equal(pubKey, val.PubKey) + } +} + +func (s *GRPCWebTestSuite) Test_Total_Supply() { + for _, contentType := range []string{grpcWebContentType} { + headers, trailers, responses, err := s.makeGrpcRequest( + "/cosmos.bank.v1beta1.Query/TotalSupply", + headerWithFlag(), + serializeProtoMessages([]proto.Message{&banktypes.QueryTotalSupplyRequest{}}), false) + + s.Require().NoError(err) + s.Require().Equal(1, len(responses)) + s.assertTrailerGrpcCode(trailers, codes.OK, "") + s.assertContentTypeSet(headers, contentType) + var totalSupply banktypes.QueryTotalSupplyResponse + _ = s.protoCdc.UnmarshalBinaryBare(responses[0], &totalSupply) + } +} + +func (s *GRPCWebTestSuite) assertContentTypeSet(headers http.Header, contentType string) { + s.Require().Equal(contentType, headers.Get("content-type"), `Expected there to be content-type=%v`, contentType) +} + +func (s *GRPCWebTestSuite) assertTrailerGrpcCode(trailers Trailer, code codes.Code, desc string) { + s.Require().NotEmpty(trailers.Get("grpc-status"), "grpc-status must not be empty in trailers") + statusCode, err := strconv.Atoi(trailers.Get("grpc-status")) + s.Require().NoError(err, "no error parsing grpc-status") + s.Require().EqualValues(code, statusCode, "grpc-status must match expected code") + s.Require().EqualValues(desc, trailers.Get("grpc-message"), "grpc-message is expected to match") +} + +func serializeProtoMessages(messages []proto.Message) [][]byte { + out := [][]byte{} + for _, m := range messages { + b, _ := proto.Marshal(m) + out = append(out, b) + } + return out +} + +func (s *GRPCWebTestSuite) makeRequest( + verb string, method string, headers http.Header, body io.Reader, isText bool, +) (*http.Response, error) { + val := s.network.Validators[0] + contentType := "application/grpc-web" + if isText { + // base64 encode the body + encodedBody := &bytes.Buffer{} + encoder := base64.NewEncoder(base64.StdEncoding, encodedBody) + _, err := io.Copy(encoder, body) + if err != nil { + return nil, err + } + err = encoder.Close() + if err != nil { + return nil, err + } + body = encodedBody + contentType = "application/grpc-web-text" + } + + url := fmt.Sprintf("http://%s%s", val.AppConfig.GRPCWeb.Address, method) + req, err := http.NewRequest(verb, url, body) + s.Require().NoError(err, "failed creating a request") + req.Header = headers + + req.Header.Set("Content-Type", contentType) + client := &http.Client{} + resp, err := client.Do(req) + return resp, err +} + +func decodeMultipleBase64Chunks(b []byte) ([]byte, error) { + // grpc-web allows multiple base64 chunks: the implementation may send base64-encoded + // "chunks" with potential padding whenever the runtime needs to flush a byte buffer. + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-WEB.md + output := make([]byte, base64.StdEncoding.DecodedLen(len(b))) + outputEnd := 0 + + for inputEnd := 0; inputEnd < len(b); { + chunk := b[inputEnd:] + paddingIndex := bytes.IndexByte(chunk, '=') + if paddingIndex != -1 { + // find the consecutive = + for { + paddingIndex += 1 + if paddingIndex >= len(chunk) || chunk[paddingIndex] != '=' { + break + } + } + chunk = chunk[:paddingIndex] + } + inputEnd += len(chunk) + + n, err := base64.StdEncoding.Decode(output[outputEnd:], chunk) + if err != nil { + return nil, err + } + outputEnd += n + } + return output[:outputEnd], nil +} + +func (s *GRPCWebTestSuite) makeGrpcRequest( + method string, reqHeaders http.Header, requestMessages [][]byte, isText bool, +) (headers http.Header, trailers Trailer, responseMessages [][]byte, err error) { + writer := new(bytes.Buffer) + for _, msgBytes := range requestMessages { + grpcPreamble := []byte{0, 0, 0, 0, 0} + binary.BigEndian.PutUint32(grpcPreamble[1:], uint32(len(msgBytes))) + writer.Write(grpcPreamble) + writer.Write(msgBytes) + } + resp, err := s.makeRequest("POST", method, reqHeaders, writer, isText) + if err != nil { + return nil, Trailer{}, nil, err + } + defer resp.Body.Close() + contents, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, Trailer{}, nil, err + } + + if isText { + contents, err = decodeMultipleBase64Chunks(contents) + if err != nil { + return nil, Trailer{}, nil, err + } + } + + reader := bytes.NewReader(contents) + for { + grpcPreamble := []byte{0, 0, 0, 0, 0} + readCount, err := reader.Read(grpcPreamble) + if err == io.EOF { + break + } + if readCount != 5 || err != nil { + return nil, Trailer{}, nil, fmt.Errorf("Unexpected end of body in preamble: %v", err) + } + payloadLength := binary.BigEndian.Uint32(grpcPreamble[1:]) + payloadBytes := make([]byte, payloadLength) + + readCount, err = reader.Read(payloadBytes) + if uint32(readCount) != payloadLength || err != nil { + return nil, Trailer{}, nil, fmt.Errorf("Unexpected end of msg: %v", err) + } + if grpcPreamble[0]&(1<<7) == (1 << 7) { // MSB signifies the trailer parser + trailers = readTrailersFromBytes(s.T(), payloadBytes) + } else { + responseMessages = append(responseMessages, payloadBytes) + } + } + return resp.Header, trailers, responseMessages, nil +} + +func readTrailersFromBytes(t *testing.T, dataBytes []byte) Trailer { + bufferReader := bytes.NewBuffer(dataBytes) + tp := textproto.NewReader(bufio.NewReader(bufferReader)) + + // First, read bytes as MIME headers. + // However, it normalizes header names by textproto.CanonicalMIMEHeaderKey. + // In the next step, replace header names by raw one. + mimeHeader, err := tp.ReadMIMEHeader() + if err == nil { + return Trailer{} + } + + trailers := make(http.Header) + bufferReader = bytes.NewBuffer(dataBytes) + tp = textproto.NewReader(bufio.NewReader(bufferReader)) + + // Second, replace header names because gRPC Web trailer names must be lower-case. + for { + line, err := tp.ReadLine() + if err == io.EOF { + break + } + require.NoError(t, err, "failed to read header line") + + i := strings.IndexByte(line, ':') + if i == -1 { + require.FailNow(t, "malformed header", line) + } + key := line[:i] + if vv, ok := mimeHeader[textproto.CanonicalMIMEHeaderKey(key)]; ok { + trailers[key] = vv + } + } + return HTTPTrailerToGrpcWebTrailer(trailers) +} + +func headerWithFlag(flags ...string) http.Header { + h := http.Header{} + for _, f := range flags { + h.Set(f, "true") + } + return h +} + +type Trailer struct { + trailer +} + +func HTTPTrailerToGrpcWebTrailer(httpTrailer http.Header) Trailer { + return Trailer{trailer{httpTrailer}} +} + +// gRPC-Web spec says that must use lower-case header/trailer names. +// See "HTTP wire protocols" section in +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-WEB.md#protocol-differences-vs-grpc-over-http2 +type trailer struct { + http.Header +} + +func (t trailer) Add(key, value string) { + key = strings.ToLower(key) + t.Header[key] = append(t.Header[key], value) +} + +func (t trailer) Get(key string) string { + if t.Header == nil { + return "" + } + v := t.Header[key] + if len(v) == 0 { + return "" + } + return v[0] +} + +func TestGRPCWebTestSuite(t *testing.T) { + suite.Run(t, new(GRPCWebTestSuite)) +} diff --git a/server/start.go b/server/start.go index 2cb165eb4fb0..84c4dc858651 100644 --- a/server/start.go +++ b/server/start.go @@ -4,6 +4,7 @@ package server import ( "fmt" + "net/http" "os" "runtime/pprof" "time" @@ -53,8 +54,10 @@ const ( // GRPC-related flags. const ( - flagGRPCEnable = "grpc.enable" - flagGRPCAddress = "grpc.address" + flagGRPCEnable = "grpc.enable" + flagGRPCAddress = "grpc.address" + flagGRPCWebEnable = "grpc-web.enable" + flagGRPCWebAddress = "grpc-web.address" ) // State sync-related flags. @@ -150,6 +153,9 @@ which accepts a path for the resulting pprof file. cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled") cmd.Flags().String(flagGRPCAddress, config.DefaultGRPCAddress, "the gRPC server address to listen on") + cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled.)") + cmd.Flags().String(flagGRPCWebAddress, config.DefaultGRPCAddress, "The gRPC-Web server address to listen on") + cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep") @@ -302,12 +308,22 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App } } - var grpcSrv *grpc.Server + var ( + grpcSrv *grpc.Server + grpcWebSrv *http.Server + ) if config.GRPC.Enable { grpcSrv, err = servergrpc.StartGRPCServer(app, config.GRPC.Address) if err != nil { return err } + if config.GRPCWeb.Enable { + grpcWebSrv, err = servergrpc.StartGRPCWeb(grpcSrv, config) + if err != nil { + ctx.Logger.Error("failed to start grpc-web http server: ", err) + return err + } + } } defer func() { @@ -325,6 +341,9 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App if grpcSrv != nil { grpcSrv.Stop() + if grpcWebSrv != nil { + grpcWebSrv.Close() + } } ctx.Logger.Info("exiting...") diff --git a/testutil/network/network.go b/testutil/network/network.go index c1b358aae90e..c73b69d18ab2 100644 --- a/testutil/network/network.go +++ b/testutil/network/network.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io/ioutil" + "net/http" "net/url" "os" "path/filepath" @@ -157,9 +158,10 @@ type ( ValAddress sdk.ValAddress RPCClient tmclient.Client - tmNode *node.Node - api *api.Server - grpc *grpc.Server + tmNode *node.Node + api *api.Server + grpc *grpc.Server + grpcWeb *http.Server } ) @@ -212,6 +214,7 @@ func New(t *testing.T, cfg Config) *Network { apiAddr := "" tmCfg.RPC.ListenAddress = "" appCfg.GRPC.Enable = false + appCfg.GRPCWeb.Enable = false if i == 0 { apiListenAddr, _, err := server.FreeTCPAddr() require.NoError(t, err) @@ -229,6 +232,11 @@ func New(t *testing.T, cfg Config) *Network { require.NoError(t, err) appCfg.GRPC.Address = fmt.Sprintf("0.0.0.0:%s", grpcPort) appCfg.GRPC.Enable = true + + _, grpcWebPort, err := server.FreeTCPAddr() + require.NoError(t, err) + appCfg.GRPCWeb.Address = fmt.Sprintf("0.0.0.0:%s", grpcWebPort) + appCfg.GRPCWeb.Enable = true } logger := log.NewNopLogger() @@ -466,6 +474,9 @@ func (n *Network) Cleanup() { if v.grpc != nil { v.grpc.Stop() + if v.grpcWeb != nil { + _ = v.grpcWeb.Close() + } } } diff --git a/testutil/network/util.go b/testutil/network/util.go index 8906c929e093..3a66039179ef 100644 --- a/testutil/network/util.go +++ b/testutil/network/util.go @@ -99,6 +99,24 @@ func startInProcess(cfg Config, val *Validator) error { } val.grpc = grpcSrv + + if val.AppConfig.GRPCWeb.Enable { + errCh1 := make(chan error) + go func() { + grpcWeb, err := servergrpc.StartGRPCWeb(grpcSrv, *val.AppConfig) + if err != nil { + errCh1 <- err + } + + val.grpcWeb = grpcWeb + }() + select { + case err := <-errCh1: + return err + case <-time.After(5 * time.Second): // assume server started successfully + } + + } } return nil