Skip to content

Commit

Permalink
tools/api: replace client to pd http client (#7574)
Browse files Browse the repository at this point in the history 
ref #7300

change tools/api client to HTTP SDK

Signed-off-by: husharp <jinhao.hu@pingcap.com>
  • Loading branch information
@HuSharp
HuSharp authored Dec 22, 2023
1 parent 16e80d8 commit 07f7e78
Show file tree
Hide file tree
Showing 6 changed files with 21 additions and 446 deletions.
13 changes: 2 additions & 11 deletions tools/pd-api-bench/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,26 +63,17 @@ The api bench cases we support are as follows:

You can run shell as follows.
```shell
go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug
go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug true
```

### HTTP params

You can use the following command to set the params of HTTP request:
```shell
go run main.go -http-cases GetMinResolvedTS-1+1 -params 'scope=cluster' -client 1 -debug
```
for more params, can use like `-params 'A=1&B=2&C=3'`


### TLS

You can use the following command to generate a certificate for testing TLS:

```shell
mkdir cert
./cert_opt.sh generate cert
go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug -cacert ./cert/ca.pem -cert ./cert/pd-server.pem -key ./cert/pd-server-key.pem
go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug true -cacert ./cert/ca.pem -cert ./cert/pd-server.pem -key ./cert/pd-server-key.pem
./cert_opt.sh cleanup cert
rm -rf cert
```
78 changes: 11 additions & 67 deletions tools/pd-api-bench/cases/cases.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,42 +19,26 @@ import (
"fmt"
"log"
"math/rand"
"net/http"
"net/url"

pd "github.com/tikv/pd/client"
"github.com/tikv/pd/pkg/statistics"
"github.com/tikv/pd/pkg/utils/apiutil"
"github.com/tikv/pd/pkg/utils/typeutil"
pdHttp "github.com/tikv/pd/client/http"
)

var (
// PDAddress is the address of PD server.
PDAddress string
// Debug is the flag to print the output of api response for debug.
Debug bool
)

var (
totalRegion int
totalStore int
storesID []uint64
)

// InitCluster initializes the cluster.
func InitCluster(ctx context.Context, cli pd.Client, httpClit *http.Client) error {
req, _ := http.NewRequestWithContext(ctx, http.MethodGet,
PDAddress+"/pd/api/v1/stats/region?start_key=&end_key=&count", http.NoBody)
resp, err := httpClit.Do(req)
if err != nil {
return err
}
statsResp := &statistics.RegionStats{}
err = apiutil.ReadJSON(resp.Body, statsResp)
func InitCluster(ctx context.Context, cli pd.Client, httpCli pdHttp.Client) error {
statsResp, err := httpCli.GetRegionStatusByKeyRange(ctx, pdHttp.NewKeyRange([]byte(""), []byte("")), false)
if err != nil {
return err
}
resp.Body.Close()
totalRegion = statsResp.Count

stores, err := cli.GetAllStores(ctx)
Expand Down Expand Up @@ -122,8 +106,7 @@ var GRPCCaseMap = map[string]GRPCCase{
// HTTPCase is the interface for all HTTP cases.
type HTTPCase interface {
Case
Do(context.Context, *http.Client) error
Params(string)
Do(context.Context, pdHttp.Client) error
}

// HTTPCaseMap is the map for all HTTP cases.
Expand All @@ -134,8 +117,6 @@ var HTTPCaseMap = map[string]HTTPCase{

type minResolvedTS struct {
*baseCase
path string
params string
}

func newMinResolvedTS() *minResolvedTS {
Expand All @@ -145,45 +126,23 @@ func newMinResolvedTS() *minResolvedTS {
qps: 1000,
burst: 1,
},
path: "/pd/api/v1/min-resolved-ts",
}
}

type minResolvedTSStruct struct {
IsRealTime bool `json:"is_real_time,omitempty"`
MinResolvedTS uint64 `json:"min_resolved_ts"`
PersistInterval typeutil.Duration `json:"persist_interval,omitempty"`
StoresMinResolvedTS map[uint64]uint64 `json:"stores_min_resolved_ts"`
}

func (c *minResolvedTS) Do(ctx context.Context, cli *http.Client) error {
url := fmt.Sprintf("%s%s", PDAddress, c.path)
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
res, err := cli.Do(req)
if err != nil {
return err
}
listResp := &minResolvedTSStruct{}
err = apiutil.ReadJSON(res.Body, listResp)
func (c *minResolvedTS) Do(ctx context.Context, cli pdHttp.Client) error {
minResolvedTS, storesMinResolvedTS, err := cli.GetMinResolvedTSByStoresIDs(ctx, storesID)
if Debug {
log.Printf("Do %s: url: %s resp: %v err: %v", c.name, url, listResp, err)
log.Printf("Do %s: minResolvedTS: %d storesMinResolvedTS: %v err: %v", c.name, minResolvedTS, storesMinResolvedTS, err)
}
if err != nil {
return err
}
res.Body.Close()
return nil
}

func (c *minResolvedTS) Params(param string) {
c.params = param
c.path = fmt.Sprintf("%s?%s", c.path, c.params)
}

type regionsStats struct {
*baseCase
regionSample int
path string
}

func newRegionStats() *regionsStats {
Expand All @@ -194,43 +153,28 @@ func newRegionStats() *regionsStats {
burst: 1,
},
regionSample: 1000,
path: "/pd/api/v1/stats/region",
}
}

func (c *regionsStats) Do(ctx context.Context, cli *http.Client) error {
func (c *regionsStats) Do(ctx context.Context, cli pdHttp.Client) error {
upperBound := totalRegion / c.regionSample
if upperBound < 1 {
upperBound = 1
}
random := rand.Intn(upperBound)
startID := c.regionSample*random*4 + 1
endID := c.regionSample*(random+1)*4 + 1
url := fmt.Sprintf("%s%s?start_key=%s&end_key=%s&%s",
PDAddress,
c.path,
url.QueryEscape(string(generateKeyForSimulator(startID, 56))),
url.QueryEscape(string(generateKeyForSimulator(endID, 56))),
"")
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
res, err := cli.Do(req)
if err != nil {
return err
}
statsResp := &statistics.RegionStats{}
err = apiutil.ReadJSON(res.Body, statsResp)
regionStats, err := cli.GetRegionStatusByKeyRange(ctx,
pdHttp.NewKeyRange(generateKeyForSimulator(startID, 56), generateKeyForSimulator(endID, 56)), false)
if Debug {
log.Printf("Do %s: url: %s resp: %v err: %v", c.name, url, statsResp, err)
log.Printf("Do %s: regionStats: %v err: %v", c.name, regionStats, err)
}
if err != nil {
return err
}
res.Body.Close()
return nil
}

func (c *regionsStats) Params(_ string) {}

type getRegion struct {
*baseCase
}
Expand Down
Empty file modified tools/pd-api-bench/cert_opt.sh
100644 → 100755
View file
Empty file.
77 changes: 1 addition & 76 deletions tools/pd-api-bench/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,113 +3,38 @@ module github.com/tools/pd-api-bench
go 1.21

require (
github.com/tikv/pd v0.0.0-00010101000000-000000000000
github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd
go.uber.org/zap v1.24.0
google.golang.org/grpc v1.54.0
)

require (
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect
github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bytedance/sonic v1.9.1 // indirect
github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect
github.com/elliotchance/pie/v2 v2.1.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.9.1 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.14.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.7.4 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect
github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect
github.com/pingcap/errcode v0.3.0 // indirect
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c // indirect
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.11.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/sasha-s/go-deadlock v0.2.0 // indirect
github.com/sirupsen/logrus v1.6.0 // indirect
github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 // indirect
github.com/soheilhy/cmux v0.1.4 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.8.3 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/unrolled/render v1.0.1 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/goleak v1.1.12 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.17.0 // indirect
golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect
golang.org/x/net v0.18.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.1.0 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)

replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0
Expand Down
Loading

0 comments on commit 07f7e78

Please sign in to comment.