api: use suitable status code and make TestResetTS stable (#7368)

close #7411

Signed-off-by: lhy1024 <admin@liudos.us>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>

pkg/mcs/tso/server/apis/v1/api.go

@@ -15,6 +15,7 @@
 package apis
 
 import (
+	"fmt"
 	"net/http"
 	"strconv"
 	"sync"
@@ -150,6 +151,7 @@ type ResetTSParams struct {
 // @Failure  400  {string}  string  "The input is invalid."
 // @Failure  403  {string}  string  "Reset ts is forbidden."
 // @Failure  500  {string}  string  "TSO server failed to proceed the request."
+// @Failure  503  {string}  string  "It's a temporary failure, please retry."
 // @Router   /admin/reset-ts [post]
 // if force-use-larger=true:
 //
@@ -185,6 +187,12 @@ func ResetTS(c *gin.Context) {
 	if err = svr.ResetTS(ts, ignoreSmaller, skipUpperBoundCheck, 0); err != nil {
 		if err == errs.ErrServerNotStarted {
 			c.String(http.StatusInternalServerError, err.Error())
+		} else if err == errs.ErrEtcdTxnConflict {
+			// If the error is ErrEtcdTxnConflict, it means there is a temporary failure.
+			// Return 503 to let the client retry.
+			// Ref: https://datatracker.ietf.org/doc/html/rfc7231#section-6.6.4
+			c.String(http.StatusServiceUnavailable,
+				fmt.Sprintf("It's a temporary failure with error %s, please retry.", err.Error()))
 		} else {
 			c.String(http.StatusForbidden, err.Error())
 		}

pkg/tso/admin.go

@@ -15,6 +15,7 @@
 package tso
 
 import (
+	"fmt"
 	"net/http"
 	"strconv"
 
@@ -53,6 +54,7 @@ func NewAdminHandler(handler Handler, rd *render.Render) *AdminHandler {
 // @Failure  400  {string}  string  "The input is invalid."
 // @Failure  403  {string}  string  "Reset ts is forbidden."
 // @Failure  500  {string}  string  "TSO server failed to proceed the request."
+// @Failure  503  {string}  string  "It's a temporary failure, please retry."
 // @Router   /admin/reset-ts [post]
 // if force-use-larger=true:
 //
@@ -96,6 +98,12 @@ func (h *AdminHandler) ResetTS(w http.ResponseWriter, r *http.Request) {
 	if err = handler.ResetTS(ts, ignoreSmaller, skipUpperBoundCheck, 0); err != nil {
 		if err == errs.ErrServerNotStarted {
 			h.rd.JSON(w, http.StatusInternalServerError, err.Error())
+		} else if err == errs.ErrEtcdTxnConflict {
+			// If the error is ErrEtcdTxnConflict, it means there is a temporary failure.
+			// Return 503 to let the client retry.
+			// Ref: https://datatracker.ietf.org/doc/html/rfc7231#section-6.6.4
+			h.rd.JSON(w, http.StatusServiceUnavailable,
+				fmt.Sprintf("It's a temporary failure with error %s, please retry.", err.Error()))
 		} else {
 			h.rd.JSON(w, http.StatusForbidden, err.Error())
 		}

server/api/admin_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"testing"
 	"time"
@@ -27,6 +28,7 @@ import (
 	"github.com/stretchr/testify/suite"
 	"github.com/tikv/pd/pkg/core"
 	"github.com/tikv/pd/pkg/replication"
+	"github.com/tikv/pd/pkg/utils/apiutil"
 	tu "github.com/tikv/pd/pkg/utils/testutil"
 	"github.com/tikv/pd/server"
 )
@@ -188,9 +190,24 @@ func (suite *adminTestSuite) TestResetTS() {
 	values, err := json.Marshal(args)
 	suite.NoError(err)
 	re := suite.Require()
-	err = tu.CheckPostJSON(testDialClient, url, values,
-		tu.StatusOK(re),
-		tu.StringEqual(re, "\"Reset ts successfully.\"\n"))
+	tu.Eventually(re, func() bool {
+		resp, err := apiutil.PostJSON(testDialClient, url, values)
+		re.NoError(err)
+		defer resp.Body.Close()
+		b, err := io.ReadAll(resp.Body)
+		re.NoError(err)
+		switch resp.StatusCode {
+		case http.StatusOK:
+			re.Contains(string(b), "Reset ts successfully.")
+			return true
+		case http.StatusServiceUnavailable:
+			re.Contains(string(b), "[PD:etcd:ErrEtcdTxnConflict]etcd transaction failed, conflicted and rolled back")
+			return false
+		default:
+			re.FailNow("unexpected status code %d", resp.StatusCode)
+			return false
+		}
+	})
 	suite.NoError(err)
 	t2 := makeTS(32 * time.Hour)
 	args["tso"] = fmt.Sprintf("%d", t2)