From 54981d1b89563547e321dbfcee24edef8a6c7a1c Mon Sep 17 00:00:00 2001 From: Hu# Date: Mon, 14 Aug 2023 19:25:59 +0800 Subject: [PATCH] api: Extend the min-resolved-ts api to support getting the specified store (#6880) close tikv/pd#6879 Nowadays we have 2 api interface for obtaining min resolved ts - /pd/api/v1/min-resolved-ts: obtain cluster's min resolved ts - /pd/api/v1/min-resolved-ts/{store_id}: obtain each store's min resolved ts For client-go's updateSafeTS, we call approach II for each store, which is not necessary. We can extend the approach I request params to obtain specified store via a list to reduce the cost per call. Signed-off-by: husharp --- client/client.go | 6 +- server/api/min_resolved_ts.go | 68 +++++++++++++++++---- server/api/min_resolved_ts_test.go | 91 +++++++++++++++++++++++++--- server/cluster/cluster.go | 23 ++++++- tests/server/cluster/cluster_test.go | 11 ++++ tools/pd-api-bench/README.md | 13 +++- tools/pd-api-bench/cases/cases.go | 40 +++++++----- tools/pd-api-bench/main.go | 8 ++- 8 files changed, 217 insertions(+), 43 deletions(-) diff --git a/client/client.go b/client/client.go index d3d3805fc4d..74cb7adf2a5 100644 --- a/client/client.go +++ b/client/client.go @@ -87,7 +87,7 @@ type Client interface { // GetRegion gets a region and its leader Peer from PD by key. // The region may expire after split. Caller is responsible for caching and // taking care of region change. - // Also it may return nil if PD finds no Region for the key temporarily, + // Also, it may return nil if PD finds no Region for the key temporarily, // client should retry later. GetRegion(ctx context.Context, key []byte, opts ...GetRegionOption) (*Region, error) // GetRegionFromMember gets a region from certain members. @@ -96,7 +96,7 @@ type Client interface { GetPrevRegion(ctx context.Context, key []byte, opts ...GetRegionOption) (*Region, error) // GetRegionByID gets a region and its leader Peer from PD by id. GetRegionByID(ctx context.Context, regionID uint64, opts ...GetRegionOption) (*Region, error) - // ScanRegion gets a list of regions, starts from the region that contains key. + // ScanRegions gets a list of regions, starts from the region that contains key. // Limit limits the maximum number of regions returned. // If a region has no leader, corresponding leader will be placed by a peer // with empty value (PeerID is 0). @@ -109,7 +109,7 @@ type Client interface { // The store may expire later. Caller is responsible for caching and taking care // of store change. GetAllStores(ctx context.Context, opts ...GetStoreOption) ([]*metapb.Store, error) - // Update GC safe point. TiKV will check it and do GC themselves if necessary. + // UpdateGCSafePoint TiKV will check it and do GC themselves if necessary. // If the given safePoint is less than the current one, it will not be updated. // Returns the new safePoint after updating. UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint64, error) diff --git a/server/api/min_resolved_ts.go b/server/api/min_resolved_ts.go index 0d30ea3395e..ef05e91b9f7 100644 --- a/server/api/min_resolved_ts.go +++ b/server/api/min_resolved_ts.go @@ -17,6 +17,7 @@ package api import ( "net/http" "strconv" + "strings" "github.com/gorilla/mux" "github.com/tikv/pd/pkg/utils/typeutil" @@ -38,17 +39,18 @@ func newMinResolvedTSHandler(svr *server.Server, rd *render.Render) *minResolved // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. type minResolvedTS struct { - IsRealTime bool `json:"is_real_time,omitempty"` - MinResolvedTS uint64 `json:"min_resolved_ts"` - PersistInterval typeutil.Duration `json:"persist_interval,omitempty"` + IsRealTime bool `json:"is_real_time,omitempty"` + MinResolvedTS uint64 `json:"min_resolved_ts"` + PersistInterval typeutil.Duration `json:"persist_interval,omitempty"` + StoresMinResolvedTS map[uint64]uint64 `json:"stores_min_resolved_ts"` } // @Tags min_store_resolved_ts // @Summary Get store-level min resolved ts. -// @Produce json -// @Success 200 {array} minResolvedTS +// @Produce json +// @Success 200 {array} minResolvedTS // @Failure 400 {string} string "The input is invalid." -// @Failure 500 {string} string "PD server failed to proceed the request." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /min-resolved-ts/{store_id} [get] func (h *minResolvedTSHandler) GetStoreMinResolvedTS(w http.ResponseWriter, r *http.Request) { c := h.svr.GetRaftCluster() @@ -67,19 +69,59 @@ func (h *minResolvedTSHandler) GetStoreMinResolvedTS(w http.ResponseWriter, r *h }) } -// @Tags min_resolved_ts -// @Summary Get cluster-level min resolved ts. +// @Tags min_resolved_ts +// @Summary Get cluster-level min resolved ts and optionally store-level min resolved ts. +// @Description Optionally, we support a query parameter `scope` +// to get store-level min resolved ts by specifying a list of store IDs. +// - When no scope is given, cluster-level's min_resolved_ts will be returned and storesMinResolvedTS will be nil. +// - When scope is `cluster`, cluster-level's min_resolved_ts will be returned and storesMinResolvedTS will be filled. +// - When scope given a list of stores, min_resolved_ts will be provided for each store +// and the scope-specific min_resolved_ts will be returned. +// // @Produce json +// @Param scope query string false "Scope of the min resolved ts: comma-separated list of store IDs (e.g., '1,2,3')." default(cluster) // @Success 200 {array} minResolvedTS // @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /min-resolved-ts [get] +// @Router /min-resolved-ts [get] func (h *minResolvedTSHandler) GetMinResolvedTS(w http.ResponseWriter, r *http.Request) { c := h.svr.GetRaftCluster() - value := c.GetMinResolvedTS() + scopeMinResolvedTS := c.GetMinResolvedTS() persistInterval := c.GetPDServerConfig().MinResolvedTSPersistenceInterval + + var storesMinResolvedTS map[uint64]uint64 + if scopeStr := r.URL.Query().Get("scope"); len(scopeStr) > 0 { + // scope is an optional parameter, it can be `cluster` or specified store IDs. + // - When no scope is given, cluster-level's min_resolved_ts will be returned and storesMinResolvedTS will be nil. + // - When scope is `cluster`, cluster-level's min_resolved_ts will be returned and storesMinResolvedTS will be filled. + // - When scope given a list of stores, min_resolved_ts will be provided for each store + // and the scope-specific min_resolved_ts will be returned. + if scopeStr == "cluster" { + stores := c.GetMetaStores() + ids := make([]uint64, len(stores)) + for i, store := range stores { + ids[i] = store.GetId() + } + // use cluster-level min_resolved_ts as the scope-specific min_resolved_ts. + _, storesMinResolvedTS = c.GetMinResolvedTSByStoreIDs(ids) + } else { + scopeIDs := strings.Split(scopeStr, ",") + ids := make([]uint64, len(scopeIDs)) + for i, idStr := range scopeIDs { + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + ids[i] = id + } + scopeMinResolvedTS, storesMinResolvedTS = c.GetMinResolvedTSByStoreIDs(ids) + } + } + h.rd.JSON(w, http.StatusOK, minResolvedTS{ - MinResolvedTS: value, - PersistInterval: persistInterval, - IsRealTime: persistInterval.Duration != 0, + MinResolvedTS: scopeMinResolvedTS, + PersistInterval: persistInterval, + IsRealTime: persistInterval.Duration != 0, + StoresMinResolvedTS: storesMinResolvedTS, }) } diff --git a/server/api/min_resolved_ts_test.go b/server/api/min_resolved_ts_test.go index 79ab71e2be1..3abc7555919 100644 --- a/server/api/min_resolved_ts_test.go +++ b/server/api/min_resolved_ts_test.go @@ -17,6 +17,8 @@ package api import ( "fmt" "reflect" + "strconv" + "strings" "testing" "time" @@ -36,6 +38,7 @@ type minResolvedTSTestSuite struct { cleanup testutil.CleanupFunc url string defaultInterval time.Duration + storesNum int } func TestMinResolvedTSTestSuite(t *testing.T) { @@ -53,11 +56,13 @@ func (suite *minResolvedTSTestSuite) SetupSuite() { suite.url = fmt.Sprintf("%s%s/api/v1/min-resolved-ts", addr, apiPrefix) mustBootstrapCluster(re, suite.svr) - mustPutStore(re, suite.svr, 1, metapb.StoreState_Up, metapb.NodeState_Serving, nil) - r1 := core.NewTestRegionInfo(7, 1, []byte("a"), []byte("b")) - mustRegionHeartbeat(re, suite.svr, r1) - r2 := core.NewTestRegionInfo(8, 1, []byte("b"), []byte("c")) - mustRegionHeartbeat(re, suite.svr, r2) + suite.storesNum = 3 + for i := 1; i <= suite.storesNum; i++ { + id := uint64(i) + mustPutStore(re, suite.svr, id, metapb.StoreState_Up, metapb.NodeState_Serving, nil) + r := core.NewTestRegionInfo(id, id, []byte(fmt.Sprintf("%da", id)), []byte(fmt.Sprintf("%db", id))) + mustRegionHeartbeat(re, suite.svr, r) + } } func (suite *minResolvedTSTestSuite) TearDownSuite() { @@ -92,9 +97,8 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTS() { PersistInterval: interval, }) // case4: set min resolved ts - rc := suite.svr.GetRaftCluster() ts := uint64(233) - rc.SetMinResolvedTS(1, ts) + suite.setAllStoresMinResolvedTS(ts) suite.checkMinResolvedTS(&minResolvedTS{ MinResolvedTS: ts, IsRealTime: true, @@ -108,7 +112,7 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTS() { IsRealTime: false, PersistInterval: interval, }) - rc.SetMinResolvedTS(1, ts+1) + suite.setAllStoresMinResolvedTS(ts) suite.checkMinResolvedTS(&minResolvedTS{ MinResolvedTS: ts, // last persist value IsRealTime: false, @@ -116,12 +120,69 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTS() { }) } +func (suite *minResolvedTSTestSuite) TestMinResolvedTSByStores() { + // run job. + interval := typeutil.Duration{Duration: suite.defaultInterval} + suite.setMinResolvedTSPersistenceInterval(interval) + suite.Eventually(func() bool { + return interval == suite.svr.GetRaftCluster().GetPDServerConfig().MinResolvedTSPersistenceInterval + }, time.Second*10, time.Millisecond*20) + // set min resolved ts. + rc := suite.svr.GetRaftCluster() + ts := uint64(233) + + // scope is `cluster` + testStoresID := make([]string, 0) + testMap := make(map[uint64]uint64) + for i := 1; i <= suite.storesNum; i++ { + storeID := uint64(i) + testTS := ts + storeID + testMap[storeID] = testTS + rc.SetMinResolvedTS(storeID, testTS) + + testStoresID = append(testStoresID, strconv.Itoa(i)) + } + suite.checkMinResolvedTSByStores(&minResolvedTS{ + MinResolvedTS: 234, + IsRealTime: true, + PersistInterval: interval, + StoresMinResolvedTS: testMap, + }, "cluster") + + // set all stores min resolved ts. + testStoresIDStr := strings.Join(testStoresID, ",") + suite.checkMinResolvedTSByStores(&minResolvedTS{ + MinResolvedTS: 234, + IsRealTime: true, + PersistInterval: interval, + StoresMinResolvedTS: testMap, + }, testStoresIDStr) + + // remove last store for test. + testStoresID = testStoresID[:len(testStoresID)-1] + testStoresIDStr = strings.Join(testStoresID, ",") + delete(testMap, uint64(suite.storesNum)) + suite.checkMinResolvedTSByStores(&minResolvedTS{ + MinResolvedTS: 234, + IsRealTime: true, + PersistInterval: interval, + StoresMinResolvedTS: testMap, + }, testStoresIDStr) +} + func (suite *minResolvedTSTestSuite) setMinResolvedTSPersistenceInterval(duration typeutil.Duration) { cfg := suite.svr.GetRaftCluster().GetPDServerConfig().Clone() cfg.MinResolvedTSPersistenceInterval = duration suite.svr.GetRaftCluster().SetPDServerConfig(cfg) } +func (suite *minResolvedTSTestSuite) setAllStoresMinResolvedTS(ts uint64) { + rc := suite.svr.GetRaftCluster() + for i := 1; i <= suite.storesNum; i++ { + rc.SetMinResolvedTS(uint64(i), ts) + } +} + func (suite *minResolvedTSTestSuite) checkMinResolvedTS(expect *minResolvedTS) { suite.Eventually(func() bool { res, err := testDialClient.Get(suite.url) @@ -130,6 +191,20 @@ func (suite *minResolvedTSTestSuite) checkMinResolvedTS(expect *minResolvedTS) { listResp := &minResolvedTS{} err = apiutil.ReadJSON(res.Body, listResp) suite.NoError(err) + suite.Nil(listResp.StoresMinResolvedTS) + return reflect.DeepEqual(expect, listResp) + }, time.Second*10, time.Millisecond*20) +} + +func (suite *minResolvedTSTestSuite) checkMinResolvedTSByStores(expect *minResolvedTS, scope string) { + suite.Eventually(func() bool { + url := fmt.Sprintf("%s?scope=%s", suite.url, scope) + res, err := testDialClient.Get(url) + suite.NoError(err) + defer res.Body.Close() + listResp := &minResolvedTS{} + err = apiutil.ReadJSON(res.Body, listResp) + suite.NoError(err) return reflect.DeepEqual(expect, listResp) }, time.Second*10, time.Millisecond*20) } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 06de6f9a56e..35bf14b617a 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -2548,10 +2548,29 @@ func (c *RaftCluster) GetMinResolvedTS() uint64 { func (c *RaftCluster) GetStoreMinResolvedTS(storeID uint64) uint64 { c.RLock() defer c.RUnlock() - if !c.isInitialized() || !core.IsAvailableForMinResolvedTS(c.GetStore(storeID)) { + store := c.GetStore(storeID) + if store == nil { + return math.MaxUint64 + } + if !c.isInitialized() || !core.IsAvailableForMinResolvedTS(store) { return math.MaxUint64 } - return c.GetStore(storeID).GetMinResolvedTS() + return store.GetMinResolvedTS() +} + +// GetMinResolvedTSByStoreIDs returns the min_resolved_ts for each store +// and returns the min_resolved_ts for all given store lists. +func (c *RaftCluster) GetMinResolvedTSByStoreIDs(ids []uint64) (uint64, map[uint64]uint64) { + minResolvedTS := uint64(math.MaxUint64) + storesMinResolvedTS := make(map[uint64]uint64) + for _, storeID := range ids { + storeTS := c.GetStoreMinResolvedTS(storeID) + storesMinResolvedTS[storeID] = storeTS + if minResolvedTS > storeTS { + minResolvedTS = storeTS + } + } + return minResolvedTS, storesMinResolvedTS } // GetExternalTS returns the external timestamp. diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index c15520aca3c..87acdf897fd 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -1301,6 +1301,13 @@ func checkMinResolvedTS(re *require.Assertions, rc *cluster.RaftCluster, expect }, time.Second*10, time.Millisecond*50) } +func checkStoreMinResolvedTS(re *require.Assertions, rc *cluster.RaftCluster, expectTS, storeID uint64) { + re.Eventually(func() bool { + ts := rc.GetStoreMinResolvedTS(storeID) + return expectTS == ts + }, time.Second*10, time.Millisecond*50) +} + func checkMinResolvedTSFromStorage(re *require.Assertions, rc *cluster.RaftCluster, expect uint64) { re.Eventually(func() bool { ts2, err := rc.GetStorage().LoadMinResolvedTS() @@ -1400,6 +1407,9 @@ func TestMinResolvedTS(t *testing.T) { resetStoreState(re, rc, store1, metapb.StoreState_Tombstone) checkMinResolvedTS(re, rc, store3TS) checkMinResolvedTSFromStorage(re, rc, store3TS) + checkStoreMinResolvedTS(re, rc, store3TS, store3) + // check no-exist store + checkStoreMinResolvedTS(re, rc, math.MaxUint64, 100) // case7: add a store with leader peer but no report min resolved ts // min resolved ts should be no change @@ -1419,6 +1429,7 @@ func TestMinResolvedTS(t *testing.T) { checkMinResolvedTS(re, rc, store3TS) setMinResolvedTSPersistenceInterval(re, rc, svr, time.Millisecond) checkMinResolvedTS(re, rc, store5TS) + checkStoreMinResolvedTS(re, rc, store5TS, store5) } // See https://github.com/tikv/pd/issues/4941 diff --git a/tools/pd-api-bench/README.md b/tools/pd-api-bench/README.md index 13b7feb6b25..0ab4ea6463b 100644 --- a/tools/pd-api-bench/README.md +++ b/tools/pd-api-bench/README.md @@ -59,11 +59,22 @@ The api bench cases we support are as follows: -debug > print the output of api response for debug +### Run Shell + You can run shell as follows. ```shell go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug ``` +### HTTP params + +You can use the following command to set the params of HTTP request: +```shell +go run main.go -http-cases GetMinResolvedTS-1+1 -params 'scope=cluster' -client 1 -debug +``` +for more params, can use like `-params 'A=1&B=2&C=3'` + + ### TLS You can use the following command to generate a certificate for testing TLS: @@ -74,4 +85,4 @@ mkdir cert go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug -cacert ./cert/ca.pem -cert ./cert/pd-server.pem -key ./cert/pd-server-key.pem ./cert_opt.sh cleanup cert rm -rf cert -``` \ No newline at end of file +``` diff --git a/tools/pd-api-bench/cases/cases.go b/tools/pd-api-bench/cases/cases.go index a3154f1462b..2b770805cd8 100644 --- a/tools/pd-api-bench/cases/cases.go +++ b/tools/pd-api-bench/cases/cases.go @@ -116,6 +116,7 @@ var GRPCCaseMap = map[string]GRPCCase{ type HTTPCase interface { Case Do(context.Context, *http.Client) error + Params(string) } var HTTPCaseMap = map[string]HTTPCase{ @@ -125,7 +126,8 @@ var HTTPCaseMap = map[string]HTTPCase{ type minResolvedTS struct { *baseCase - path string + path string + params string } func newMinResolvedTS() *minResolvedTS { @@ -140,14 +142,15 @@ func newMinResolvedTS() *minResolvedTS { } type minResolvedTSStruct struct { - IsRealTime bool `json:"is_real_time,omitempty"` - MinResolvedTS uint64 `json:"min_resolved_ts"` - PersistInterval typeutil.Duration `json:"persist_interval,omitempty"` + IsRealTime bool `json:"is_real_time,omitempty"` + MinResolvedTS uint64 `json:"min_resolved_ts"` + PersistInterval typeutil.Duration `json:"persist_interval,omitempty"` + StoresMinResolvedTS map[uint64]uint64 `json:"stores_min_resolved_ts"` } func (c *minResolvedTS) Do(ctx context.Context, cli *http.Client) error { - storeIdx := rand.Intn(int(totalStore)) - req, _ := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s%s/%d", PDAddress, c.path, storesID[storeIdx]), nil) + url := fmt.Sprintf("%s%s", PDAddress, c.path) + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) res, err := cli.Do(req) if err != nil { return err @@ -155,7 +158,7 @@ func (c *minResolvedTS) Do(ctx context.Context, cli *http.Client) error { listResp := &minResolvedTSStruct{} err = apiutil.ReadJSON(res.Body, listResp) if Debug { - log.Printf("Do %s: %v %v", c.name, listResp, err) + log.Printf("Do %s: url: %s resp: %v err: %v", c.name, url, listResp, err) } if err != nil { return err @@ -164,6 +167,11 @@ func (c *minResolvedTS) Do(ctx context.Context, cli *http.Client) error { return nil } +func (c *minResolvedTS) Params(param string) { + c.params = param + c.path = fmt.Sprintf("%s?%s", c.path, c.params) +} + type regionsStats struct { *baseCase regionSample int @@ -183,20 +191,20 @@ func newRegionStats() *regionsStats { } func (c *regionsStats) Do(ctx context.Context, cli *http.Client) error { - upperBound := int(totalRegion) / c.regionSample + upperBound := totalRegion / c.regionSample if upperBound < 1 { upperBound = 1 } random := rand.Intn(upperBound) startID := c.regionSample*random*4 + 1 endID := c.regionSample*(random+1)*4 + 1 - req, _ := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s%s?start_key=%s&end_key=%s&%s", + url := fmt.Sprintf("%s%s?start_key=%s&end_key=%s&%s", PDAddress, c.path, url.QueryEscape(string(generateKeyForSimulator(startID, 56))), url.QueryEscape(string(generateKeyForSimulator(endID, 56))), - "", - ), nil) + "") + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) res, err := cli.Do(req) if err != nil { return err @@ -204,7 +212,7 @@ func (c *regionsStats) Do(ctx context.Context, cli *http.Client) error { statsResp := &statistics.RegionStats{} err = apiutil.ReadJSON(res.Body, statsResp) if Debug { - log.Printf("Do %s: %v %v", c.name, statsResp, err) + log.Printf("Do %s: url: %s resp: %v err: %v", c.name, url, statsResp, err) } if err != nil { return err @@ -213,6 +221,8 @@ func (c *regionsStats) Do(ctx context.Context, cli *http.Client) error { return nil } +func (c *regionsStats) Params(_ string) {} + type getRegion struct { *baseCase } @@ -228,7 +238,7 @@ func newGetRegion() *getRegion { } func (c *getRegion) Unary(ctx context.Context, cli pd.Client) error { - id := rand.Intn(int(totalRegion))*4 + 1 + id := rand.Intn(totalRegion)*4 + 1 _, err := cli.GetRegion(ctx, generateKeyForSimulator(id, 56)) if err != nil { return err @@ -253,7 +263,7 @@ func newScanRegions() *scanRegions { } func (c *scanRegions) Unary(ctx context.Context, cli pd.Client) error { - upperBound := int(totalRegion) / c.regionSample + upperBound := totalRegion / c.regionSample random := rand.Intn(upperBound) startID := c.regionSample*random*4 + 1 endID := c.regionSample*(random+1)*4 + 1 @@ -279,7 +289,7 @@ func newGetStore() *getStore { } func (c *getStore) Unary(ctx context.Context, cli pd.Client) error { - storeIdx := rand.Intn(int(totalStore)) + storeIdx := rand.Intn(totalStore) _, err := cli.GetStore(ctx, storesID[storeIdx]) if err != nil { return err diff --git a/tools/pd-api-bench/main.go b/tools/pd-api-bench/main.go index 7032ef1df00..a891f7d2318 100644 --- a/tools/pd-api-bench/main.go +++ b/tools/pd-api-bench/main.go @@ -48,13 +48,16 @@ var ( qps = flag.Int64("qps", 1000, "qps") burst = flag.Int64("burst", 1, "burst") + // http params + httpParams = flag.String("params", "", "http params") + // tls caPath = flag.String("cacert", "", "path of file that contains list of trusted SSL CAs") certPath = flag.String("cert", "", "path of file that contains X509 certificate in PEM format") keyPath = flag.String("key", "", "path of file that contains X509 key in PEM format") ) -var base int64 = int64(time.Second) / int64(time.Microsecond) +var base = int64(time.Second) / int64(time.Microsecond) func main() { flag.Parse() @@ -216,6 +219,9 @@ func handleHTTPCase(ctx context.Context, hcase cases.HTTPCase, httpClis []*http. burst := hcase.GetBurst() tt := time.Duration(base/qps*burst*int64(*client)) * time.Microsecond log.Printf("begin to run http case %s, with qps = %d and burst = %d, interval is %v", hcase.Name(), qps, burst, tt) + if *httpParams != "" { + hcase.Params(*httpParams) + } for _, hCli := range httpClis { go func(hCli *http.Client) { var ticker = time.NewTicker(tt)