Skip to content

Commit

Permalink
apiv2: keyspace management (#5313)
Browse files Browse the repository at this point in the history
ref #5265, ref #5291, ref #5293, ref #5297

Add APIv2 HTTP interface for keyspace management.

Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com>

Co-authored-by: Ti Chi Robot <ti-community-prow-bot@tidb.io>
  • Loading branch information
AmoebaProtozoa and ti-chi-bot authored Feb 2, 2023
1 parent 8830739 commit abb62d2
Show file tree
Hide file tree
Showing 4 changed files with 627 additions and 22 deletions.
341 changes: 341 additions & 0 deletions server/apiv2/handlers/keyspace.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,341 @@
// Copyright 2023 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package handlers

import (
"encoding/json"
"net/http"
"strconv"
"strings"
"time"

"github.com/gin-gonic/gin"
"github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/keyspacepb"
"github.com/tikv/pd/pkg/errs"
"github.com/tikv/pd/server"
"github.com/tikv/pd/server/apiv2/middlewares"
"github.com/tikv/pd/server/keyspace"
)

// RegisterKeyspace register keyspace related handlers to router paths.
func RegisterKeyspace(r *gin.RouterGroup) {
router := r.Group("keyspaces")
router.Use(middlewares.BootstrapChecker())
router.POST("", CreateKeyspace)
router.GET("", LoadAllKeyspaces)
router.GET("/:name", LoadKeyspace)
router.PATCH("/:name/config", UpdateKeyspaceConfig)
router.PUT("/:name/state", UpdateKeyspaceState)
}

// CreateKeyspaceParams represents parameters needed when creating a new keyspace.
// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it.
type CreateKeyspaceParams struct {
Name string `json:"name"`
Config map[string]string `json:"config"`
}

// CreateKeyspace creates keyspace according to given input.
// @Tags keyspaces
// @Summary Create new keyspace.
// @Param body body CreateKeyspaceParams true "Create keyspace parameters"
// @Produce json
// @Success 200 {object} KeyspaceMeta
// @Failure 400 {string} string "The input is invalid."
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /keyspaces [post]
func CreateKeyspace(c *gin.Context) {
svr := c.MustGet("server").(*server.Server)
manager := svr.GetKeyspaceManager()
createParams := &CreateKeyspaceParams{}
err := c.BindJSON(createParams)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause())
return
}
req := &keyspace.CreateKeyspaceRequest{
Name: createParams.Name,
Config: createParams.Config,
Now: time.Now().Unix(),
}
meta, err := manager.CreateKeyspace(req)
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
return
}
c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta})
}

// LoadKeyspace returns target keyspace.
// @Tags keyspaces
// @Summary Get keyspace info.
// @Param name path string true "Keyspace Name"
// @Produce json
// @Success 200 {object} KeyspaceMeta
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /keyspaces/{name} [get]
func LoadKeyspace(c *gin.Context) {
svr := c.MustGet("server").(*server.Server)
manager := svr.GetKeyspaceManager()
name := c.Param("name")
meta, err := manager.LoadKeyspace(name)
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
return
}
c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta})
}

// parseLoadAllQuery parses LoadAllKeyspaces' query parameters.
// page_token:
// The keyspace id of the scan start. If not set, scan from keyspace with id 1.
// It's string of spaceID of the previous scan result's last element (next_page_token).
// limit:
// The maximum number of keyspace metas to return. If not set, no limit is posed.
// Every scan scans limit + 1 keyspaces (if limit != 0), the extra scanned keyspace
// is to check if there's more, and used to set next_page_token in response.
func parseLoadAllQuery(c *gin.Context) (scanStart uint32, scanLimit int, err error) {
pageToken, set := c.GetQuery("page_token")
if !set || pageToken == "" {
// If pageToken is empty or unset, then scan from spaceID of 1.
scanStart = 0
} else {
scanStart64, err := strconv.ParseUint(pageToken, 10, 32)
if err != nil {
return 0, 0, err
}
scanStart = uint32(scanStart64)
}

limitStr, set := c.GetQuery("limit")
if !set || limitStr == "" || limitStr == "0" {
// If limit is unset or empty or 0, then no limit is posed for scan.
scanLimit = 0
} else {
scanLimit64, err := strconv.ParseInt(limitStr, 10, 64)
if err != nil {
return 0, 0, err
}
// Scan an extra element for next_page_token.
scanLimit = int(scanLimit64) + 1
}

return scanStart, scanLimit, nil
}

// LoadAllKeyspacesResponse represents response given when loading all keyspaces.
// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it.
type LoadAllKeyspacesResponse struct {
Keyspaces []*KeyspaceMeta `json:"keyspaces"`
// Token that can be used to read immediate next page.
// If it's empty, then end has been reached.
NextPageToken string `json:"next_page_token"`
}

// LoadAllKeyspaces loads range of keyspaces.
// @Tags keyspaces
// @Summary list keyspaces.
// @Param page_token query string false "page token"
// @Param limit query string false "maximum number of results to return"
// @Produce json
// @Success 200 {object} LoadAllKeyspacesResponse
// @Failure 400 {string} string "The input is invalid."
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /keyspaces [get]
func LoadAllKeyspaces(c *gin.Context) {
svr := c.MustGet("server").(*server.Server)
manager := svr.GetKeyspaceManager()
scanStart, scanLimit, err := parseLoadAllQuery(c)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
return
}
scanned, err := manager.LoadRangeKeyspace(scanStart, scanLimit)
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
return
}
resp := &LoadAllKeyspacesResponse{}
// If scanned 0 keyspaces, return result immediately.
if len(scanned) == 0 {
c.IndentedJSON(http.StatusOK, resp)
return
}
var resultKeyspaces []*KeyspaceMeta
if scanLimit == 0 || len(scanned) < scanLimit {
// No next page, all scanned are results.
resultKeyspaces = make([]*KeyspaceMeta, len(scanned))
for i, meta := range scanned {
resultKeyspaces[i] = &KeyspaceMeta{meta}
}
} else {
// Scanned limit + 1 keyspaces, there is next page, all but last are results.
resultKeyspaces = make([]*KeyspaceMeta, len(scanned)-1)
for i := range resultKeyspaces {
resultKeyspaces[i] = &KeyspaceMeta{scanned[i]}
}
// Also set next_page_token here.
resp.NextPageToken = strconv.Itoa(int(scanned[len(scanned)-1].Id))
}
resp.Keyspaces = resultKeyspaces
c.IndentedJSON(http.StatusOK, resp)
}

// UpdateConfigParams represents parameters needed to modify target keyspace's configs.
// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it.
// A Map of string to string pointer is used to differentiate between json null and "",
// which will both be set to "" if value type is string during binding.
type UpdateConfigParams struct {
Config map[string]*string `json:"config"`
}

// UpdateKeyspaceConfig updates target keyspace's config.
// This api uses PATCH semantic and supports JSON Merge Patch.
// format and processing rules.
// @Tags keyspaces
// @Summary Update keyspace config.
// @Param name path string true "Keyspace Name"
// @Param body body UpdateConfigParams true "Update keyspace parameters"
// @Produce json
// @Success 200 {object} KeyspaceMeta
// @Failure 400 {string} string "The input is invalid."
// @Failure 500 {string} string "PD server failed to proceed the request."
// Router /keyspaces/{name}/config [patch]
func UpdateKeyspaceConfig(c *gin.Context) {
svr := c.MustGet("server").(*server.Server)
manager := svr.GetKeyspaceManager()
name := c.Param("name")
configParams := &UpdateConfigParams{}
err := c.BindJSON(configParams)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause())
return
}
mutations := getMutations(configParams.Config)
meta, err := manager.UpdateKeyspaceConfig(name, mutations)
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
return
}
c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta})
}

// getMutations converts a given JSON merge patch to a series of keyspace config mutations.
func getMutations(patch map[string]*string) []*keyspace.Mutation {
mutations := make([]*keyspace.Mutation, 0, len(patch))
for k, v := range patch {
if v == nil {
mutations = append(mutations, &keyspace.Mutation{
Op: keyspace.OpDel,
Key: k,
})
} else {
mutations = append(mutations, &keyspace.Mutation{
Op: keyspace.OpPut,
Key: k,
Value: *v,
})
}
}
return mutations
}

// UpdateStateParam represents parameters needed to modify target keyspace's state.
// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it.
type UpdateStateParam struct {
State string `json:"state"`
}

// UpdateKeyspaceState update the target keyspace's state.
// @Tags keyspaces
// @Summary Update keyspace state.
// @Param name path string true "Keyspace Name"
// @Param body body UpdateStateParam true "New state for the keyspace"
// @Produce json
// @Success 200 {object} KeyspaceMeta
// @Failure 400 {string} string "The input is invalid."
// @Failure 500 {string} string "PD server failed to proceed the request."
// Router /keyspaces/{name}/state [put]
func UpdateKeyspaceState(c *gin.Context) {
svr := c.MustGet("server").(*server.Server)
manager := svr.GetKeyspaceManager()
name := c.Param("name")
param := &UpdateStateParam{}
err := c.BindJSON(param)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause())
return
}
targetState, ok := keyspacepb.KeyspaceState_value[strings.ToUpper(param.State)]
if !ok {
c.AbortWithStatusJSON(http.StatusBadRequest, errors.Errorf("unknown target state: %s", param.State))
return
}
meta, err := manager.UpdateKeyspaceState(name, keyspacepb.KeyspaceState(targetState), time.Now().Unix())
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
return
}
c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta})
}

// KeyspaceMeta wraps keyspacepb.KeyspaceMeta to provide custom JSON marshal.
type KeyspaceMeta struct {
*keyspacepb.KeyspaceMeta
}

// MarshalJSON creates custom marshal of KeyspaceMeta with the following:
// 1. Keyspace ID are removed from marshal result to avoid exposure of internal mechanics.
// 2. Keyspace State are marshaled to their corresponding name for better readability.
func (meta *KeyspaceMeta) MarshalJSON() ([]byte, error) {
return json.Marshal(&struct {
Name string `json:"name,omitempty"`
State string `json:"state,omitempty"`
CreatedAt int64 `json:"created_at,omitempty"`
StateChangedAt int64 `json:"state_changed_at,omitempty"`
Config map[string]string `json:"config,omitempty"`
}{
meta.Name,
meta.State.String(),
meta.CreatedAt,
meta.StateChangedAt,
meta.Config,
})
}

// UnmarshalJSON reverse KeyspaceMeta's the Custom JSON marshal.
func (meta *KeyspaceMeta) UnmarshalJSON(data []byte) error {
aux := &struct {
Name string `json:"name,omitempty"`
State string `json:"state,omitempty"`
CreatedAt int64 `json:"created_at,omitempty"`
StateChangedAt int64 `json:"state_changed_at,omitempty"`
Config map[string]string `json:"config,omitempty"`
}{}

if err := json.Unmarshal(data, aux); err != nil {
return err
}
pbMeta := &keyspacepb.KeyspaceMeta{
Name: aux.Name,
State: keyspacepb.KeyspaceState(keyspacepb.KeyspaceState_value[aux.State]),
CreatedAt: aux.CreatedAt,
StateChangedAt: aux.StateChangedAt,
Config: aux.Config,
}
meta.KeyspaceMeta = pbMeta
return nil
}
14 changes: 12 additions & 2 deletions server/apiv2/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/joho/godotenv"
"github.com/tikv/pd/server"
"github.com/tikv/pd/server/apiv2/handlers"
"github.com/tikv/pd/server/apiv2/middlewares"
)

Expand All @@ -37,6 +38,15 @@ var group = server.APIServiceGroup{
const apiV2Prefix = "/pd/api/v2/"

// NewV2Handler creates a HTTP handler for API.
// @title Placement Driver Core API
// @version 2.0
// @description This is placement driver.
// @contact.name Placement Driver Support
// @contact.url https://github.com/tikv/pd/issues
// @contact.email info@pingcap.com
// @license.name Apache 2.0
// @license.url http://www.apache.org/licenses/LICENSE-2.0.html
// @BasePath /pd/api/v2
func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, server.APIServiceGroup, error) {
once.Do(func() {
// See https://github.com/pingcap/tidb-dashboard/blob/f8ecb64e3d63f4ed91c3dca7a04362418ade01d8/pkg/apiserver/apiserver.go#L84
Expand All @@ -50,7 +60,7 @@ func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, server.A
c.Next()
})
router.Use(middlewares.Redirector())
_ = router.Group(apiV2Prefix)

root := router.Group(apiV2Prefix)
handlers.RegisterKeyspace(root)
return router, group, nil
}
Loading

0 comments on commit abb62d2

Please sign in to comment.