Skip to content

Commit

Permalink
Merge branch 'master' into remove-uuid
Browse files Browse the repository at this point in the history
  • Loading branch information
@ti-chi-bot
ti-chi-bot[bot] authored Mar 7, 2024
2 parents 7fd978a + 2574aaa commit 00d9179
Show file tree
Hide file tree
Showing 6 changed files with 184 additions and 12 deletions.
13 changes: 13 additions & 0 deletions client/http/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ const (
operators = "/pd/api/v1/operators"
// Micro Service
microServicePrefix = "/pd/api/v2/ms"
// Keyspace
KeyspaceConfig = "/pd/api/v2/keyspaces/%s/config"
GetKeyspaceMetaByName = "/pd/api/v2/keyspaces/%s"
)

// RegionByID returns the path of PD HTTP API to get region by ID.
Expand Down Expand Up @@ -201,3 +204,13 @@ func MicroServiceMembers(service string) string {
func MicroServicePrimary(service string) string {
return fmt.Sprintf("%s/primary/%s", microServicePrefix, service)
}

// GetUpdateKeyspaceConfigURL returns the path of PD HTTP API to update keyspace config.
func GetUpdateKeyspaceConfigURL(keyspaceName string) string {
return fmt.Sprintf(KeyspaceConfig, keyspaceName)
}

// GetKeyspaceMetaByNameURL returns the path of PD HTTP API to get keyspace meta by keyspace name.
func GetKeyspaceMetaByNameURL(keyspaceName string) string {
return fmt.Sprintf(GetKeyspaceMetaByName, keyspaceName)
}
50 changes: 50 additions & 0 deletions client/http/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"strings"

"github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/keyspacepb"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/kvproto/pkg/pdpb"
"github.com/tikv/pd/client/retry"
Expand Down Expand Up @@ -95,6 +96,10 @@ type Client interface {
GetMicroServicePrimary(context.Context, string) (string, error)
DeleteOperators(context.Context) error

/* Keyspace interface */
UpdateKeyspaceSafePointVersion(ctx context.Context, keyspaceName string, keyspaceSafePointVersion *KeyspaceSafePointVersionConfig) error
GetKeyspaceMetaByName(ctx context.Context, keyspaceName string) (*keyspacepb.KeyspaceMeta, error)

/* Client-related methods */
// WithCallerID sets and returns a new client with the given caller ID.
WithCallerID(string) Client
Expand Down Expand Up @@ -900,3 +905,48 @@ func (c *client) DeleteOperators(ctx context.Context) error {
WithURI(operators).
WithMethod(http.MethodDelete))
}

// UpdateKeyspaceSafePointVersion patches the keyspace config.
func (c *client) UpdateKeyspaceSafePointVersion(ctx context.Context, keyspaceName string, keyspaceSafePointVersion *KeyspaceSafePointVersionConfig) error {
keyspaceConfigPatchJSON, err := json.Marshal(keyspaceSafePointVersion)
if err != nil {
return errors.Trace(err)
}
return c.request(ctx, newRequestInfo().
WithName(UpdateKeyspaceSafePointVersionName).
WithURI(GetUpdateKeyspaceConfigURL(keyspaceName)).
WithMethod(http.MethodPatch).
WithBody(keyspaceConfigPatchJSON))
}

// GetKeyspaceMetaByName get the given keyspace meta.
func (c *client) GetKeyspaceMetaByName(ctx context.Context, keyspaceName string) (*keyspacepb.KeyspaceMeta, error) {
var (
tempKeyspaceMeta tempKeyspaceMeta
keyspaceMetaPB keyspacepb.KeyspaceMeta
)
err := c.request(ctx, newRequestInfo().
WithName(GetKeyspaceMetaByNameName).
WithURI(GetKeyspaceMetaByNameURL(keyspaceName)).
WithMethod(http.MethodGet).
WithResp(&tempKeyspaceMeta))

if err != nil {
return nil, err
}

keyspaceState, err := stringToKeyspaceState(tempKeyspaceMeta.State)
if err != nil {
return nil, err
}

keyspaceMetaPB = keyspacepb.KeyspaceMeta{
Name: tempKeyspaceMeta.Name,
Id: tempKeyspaceMeta.ID,
Config: tempKeyspaceMeta.Config,
CreatedAt: tempKeyspaceMeta.CreatedAt,
StateChangedAt: tempKeyspaceMeta.StateChangedAt,
State: keyspaceState,
}
return &keyspaceMetaPB, nil
}
2 changes: 2 additions & 0 deletions client/http/request_info.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ const (
setSnapshotRecoveringMarkName = "SetSnapshotRecoveringMark"
deleteSnapshotRecoveringMarkName = "DeleteSnapshotRecoveringMark"
deleteOperators = "DeleteOperators"
UpdateKeyspaceSafePointVersionName = "UpdateKeyspaceSafePointVersion"
GetKeyspaceMetaByNameName = "GetKeyspaceMetaByName"
)

type requestInfo struct {
Expand Down
37 changes: 37 additions & 0 deletions client/http/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@ package http
import (
"encoding/hex"
"encoding/json"
"fmt"
"net/url"
"time"

"github.com/pingcap/kvproto/pkg/encryptionpb"
"github.com/pingcap/kvproto/pkg/keyspacepb"
"github.com/pingcap/kvproto/pkg/pdpb"
)

Expand Down Expand Up @@ -601,3 +603,38 @@ type MicroServiceMember struct {
DeployPath string `json:"deploy-path"`
StartTimestamp int64 `json:"start-timestamp"`
}

// KeyspaceSafePointVersion represents parameters needed to modify the safe point version.
type KeyspaceSafePointVersion struct {
SafePointVersion string `json:"safe_point_version,omitempty"`
}

// KeyspaceSafePointVersionConfig represents parameters needed to modify target keyspace's configs.
type KeyspaceSafePointVersionConfig struct {
Config KeyspaceSafePointVersion `json:"config"`
}

// tempKeyspaceMeta is the keyspace meta struct that returned from the http interface.
type tempKeyspaceMeta struct {
ID uint32 `json:"id"`
Name string `json:"name"`
State string `json:"state"`
CreatedAt int64 `json:"created_at"`
StateChangedAt int64 `json:"state_changed_at"`
Config map[string]string `json:"config"`
}

func stringToKeyspaceState(str string) (keyspacepb.KeyspaceState, error) {
switch str {
case "ENABLED":
return keyspacepb.KeyspaceState_ENABLED, nil
case "DISABLED":
return keyspacepb.KeyspaceState_DISABLED, nil
case "ARCHIVED":
return keyspacepb.KeyspaceState_ARCHIVED, nil
case "TOMBSTONE":
return keyspacepb.KeyspaceState_TOMBSTONE, nil
default:
return keyspacepb.KeyspaceState(0), fmt.Errorf("invalid KeyspaceState string: %s", str)
}
}
67 changes: 55 additions & 12 deletions metrics/grafana/pd.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1809,7 +1809,7 @@
"steppedLine": false,
"targets": [
{
"expr": "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}[30s])",
"expr": "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}[30s])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
Expand All @@ -1819,7 +1819,7 @@
},
{
"exemplar": true,
"expr": "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
"expr": "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}",
"hide": false,
"interval": "",
"legendFormat": "{{job}}-{{instance}}-limit",
Expand Down Expand Up @@ -1917,7 +1917,7 @@
"steppedLine": false,
"targets": [
{
"expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
"expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}",
"format": "time_series",
"hide": false,
"interval": "",
Expand All @@ -1927,42 +1927,42 @@
"step": 4
},
{
"expr": "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
"expr": "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}",
"hide": true,
"interval": "",
"legendFormat": "HeapSys-{{job}}-{{instance}}",
"refId": "B"
},
{
"expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
"expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}",
"hide": false,
"interval": "",
"legendFormat": "HeapInuse-{{job}}-{{instance}}",
"refId": "C"
},
{
"expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
"expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}",
"hide": true,
"interval": "",
"legendFormat": "HeapAlloc-{{job}}-{{instance}}",
"refId": "D"
},
{
"expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
"expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}",
"hide": true,
"interval": "",
"legendFormat": "HeapIdle-{{job}}-{{instance}}",
"refId": "E"
},
{
"expr": "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
"expr": "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}",
"hide": true,
"interval": "",
"legendFormat": "HeapReleased-{{job}}-{{instance}}",
"refId": "F"
},
{
"expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
"expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}",
"hide": true,
"interval": "",
"legendFormat": "GCTrigger-{{job}}-{{instance}}",
Expand Down Expand Up @@ -2059,7 +2059,7 @@
"steppedLine": false,
"targets": [
{
"expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"})",
"expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"})",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
Expand Down Expand Up @@ -2158,7 +2158,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}",
"expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
Expand Down Expand Up @@ -10618,6 +10618,40 @@
"intervalFactor": 1,
"legendFormat": "99.999% tso",
"refId": "D"
},
{
"expr": "histogram_quantile(0.90, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
"legendFormat": "90% tso",
"refId": "E",
"step": 2
},
{
"expr": "histogram_quantile(0.99, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
"legendFormat": "99% tso",
"refId": "F",
"step": 2
},
{
"expr": "histogram_quantile(0.999, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
"legendFormat": "99.9% tso",
"refId": "G",
"step": 2
},
{
"expr": "histogram_quantile(0.99999, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "99.999% tso",
"refId": "H"
}
],
"thresholds": [],
Expand Down Expand Up @@ -10830,6 +10864,15 @@
"hide": true,
"refId": "B",
"step": 2
},
{
"expr": "sum(rate(tso_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "tso request/secs",
"refId": "C",
"step": 2
}
],
"thresholds": [],
Expand Down Expand Up @@ -13590,4 +13633,4 @@
"title": "Test-Cluster-PD",
"uid": "Q6RuHYIWk",
"version": 1
}
}
27 changes: 27 additions & 0 deletions tests/integrations/client/http_client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func (suite *httpClientTestSuite) SetupSuite() {
leader := cluster.WaitLeader()
re.NotEmpty(leader)
leaderServer := cluster.GetLeaderServer()

err = leaderServer.BootstrapCluster()
re.NoError(err)
for _, region := range []*core.RegionInfo{
Expand Down Expand Up @@ -751,3 +752,29 @@ func (suite *httpClientTestSuite) TestRedirectWithMetrics() {
re.Equal(float64(3), out.Counter.GetValue())
c.Close()
}

func (suite *httpClientTestSuite) TestUpdateKeyspaceSafePointVersion() {
suite.RunTestInTwoModes(suite.checkUpdateKeyspaceSafePointVersion)
}

func (suite *httpClientTestSuite) checkUpdateKeyspaceSafePointVersion(mode mode, client pd.Client) {
re := suite.Require()
env := suite.env[mode]

keyspaceName := "DEFAULT"
safePointVersion := "v2"

keyspaceSafePointVersionConfig := pd.KeyspaceSafePointVersionConfig{
Config: pd.KeyspaceSafePointVersion{
SafePointVersion: safePointVersion,
},
}
err := client.UpdateKeyspaceSafePointVersion(env.ctx, keyspaceName, &keyspaceSafePointVersionConfig)
re.NoError(err)

keyspaceMetaRes, err := client.GetKeyspaceMetaByName(env.ctx, keyspaceName)
re.NoError(err)
val, ok := keyspaceMetaRes.Config["safe_point_version"]
re.True(ok)
re.Equal(safePointVersion, val)
}

0 comments on commit 00d9179

Please sign in to comment.