From a1410edc8e08989e0291bcca8bcb5e69859ad5a0 Mon Sep 17 00:00:00 2001 From: Hu# Date: Fri, 22 Sep 2023 18:28:43 +0800 Subject: [PATCH] This is an automated cherry-pick of #7087 close tikv/pd#7094 Signed-off-by: ti-chi-bot --- Makefile | 12 +++++ server/api/admin.go | 5 +- server/api/admin_test.go | 10 +++- server/api/plugin.go | 3 ++ server/api/plugin_disable.go | 41 ++++++++++++++++ server/api/server_test.go | 23 +++++++++ server/handler.go | 8 ++++ server/replication/replication_mode.go | 5 +- server/server.go | 21 ++++++++- server/server_test.go | 33 +++++++++++++ server/util.go | 65 ++++++++++++++++++++++++++ 11 files changed, 220 insertions(+), 6 deletions(-) create mode 100644 server/api/plugin_disable.go diff --git a/Makefile b/Makefile index 99d6d680a7a5..605f2359840e 100644 --- a/Makefile +++ b/Makefile @@ -38,11 +38,23 @@ ifeq ("$(WITH_RACE)", "1") BUILD_CGO_ENABLED := 1 endif +<<<<<<< HEAD LDFLAGS += -X "$(PD_PKG)/server/versioninfo.PDReleaseVersion=$(shell git describe --tags --dirty --always)" LDFLAGS += -X "$(PD_PKG)/server/versioninfo.PDBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')" LDFLAGS += -X "$(PD_PKG)/server/versioninfo.PDGitHash=$(shell git rev-parse HEAD)" LDFLAGS += -X "$(PD_PKG)/server/versioninfo.PDGitBranch=$(shell git rev-parse --abbrev-ref HEAD)" LDFLAGS += -X "$(PD_PKG)/server/versioninfo.PDEdition=$(PD_EDITION)" +======= +ifeq ($(PLUGIN), 1) + BUILD_TAGS += with_plugin +endif + +LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDReleaseVersion=$(shell git describe --tags --dirty --always)" +LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')" +LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDGitHash=$(shell git rev-parse HEAD)" +LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDGitBranch=$(shell git rev-parse --abbrev-ref HEAD)" +LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDEdition=$(PD_EDITION)" +>>>>>>> a21fd58d9 (security: disable plugin in default and persist file in specified dir (#7087)) ifneq ($(DASHBOARD), 0) # Note: LDFLAGS must be evaluated lazily for these scripts to work correctly diff --git a/server/api/admin.go b/server/api/admin.go index 93dffbd66c5c..f29af732bd8c 100644 --- a/server/api/admin.go +++ b/server/api/admin.go @@ -133,7 +133,10 @@ func (h *adminHandler) ResetTS(w http.ResponseWriter, r *http.Request) { } // Intentionally no swagger mark as it is supposed to be only used in -// server-to-server. For security reason, it only accepts JSON formatted data. +// server-to-server. +// For security reason, +// - it only accepts JSON formatted data. +// - it only accepts file name which is `DrStatusFile`. func (h *adminHandler) SavePersistFile(w http.ResponseWriter, r *http.Request) { data, err := io.ReadAll(r.Body) if err != nil { diff --git a/server/api/admin_test.go b/server/api/admin_test.go index 6051edf18053..eb16a90e915f 100644 --- a/server/api/admin_test.go +++ b/server/api/admin_test.go @@ -25,8 +25,14 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/suite" +<<<<<<< HEAD "github.com/tikv/pd/pkg/apiutil" tu "github.com/tikv/pd/pkg/testutil" +======= + "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/replication" + tu "github.com/tikv/pd/pkg/utils/testutil" +>>>>>>> a21fd58d9 (security: disable plugin in default and persist file in specified dir (#7087)) "github.com/tikv/pd/server" "github.com/tikv/pd/server/core" ) @@ -168,10 +174,10 @@ func (suite *adminTestSuite) TestDropRegions() { func (suite *adminTestSuite) TestPersistFile() { data := []byte("#!/bin/sh\nrm -rf /") re := suite.Require() - err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/admin/persist-file/fun.sh", data, tu.StatusNotOK(re)) + err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/admin/persist-file/"+replication.DrStatusFile, data, tu.StatusNotOK(re)) suite.NoError(err) data = []byte(`{"foo":"bar"}`) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/admin/persist-file/good.json", data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/admin/persist-file/"+replication.DrStatusFile, data, tu.StatusOK(re)) suite.NoError(err) } diff --git a/server/api/plugin.go b/server/api/plugin.go index 16894304e9be..8727af021158 100644 --- a/server/api/plugin.go +++ b/server/api/plugin.go @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build with_plugin +// +build with_plugin + package api import ( diff --git a/server/api/plugin_disable.go b/server/api/plugin_disable.go new file mode 100644 index 000000000000..2676dbb91e20 --- /dev/null +++ b/server/api/plugin_disable.go @@ -0,0 +1,41 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !with_plugin +// +build !with_plugin + +package api + +import ( + "net/http" + + "github.com/tikv/pd/server" + "github.com/unrolled/render" +) + +type pluginHandler struct{} + +func newPluginHandler(_ *server.Handler, _ *render.Render) *pluginHandler { + return &pluginHandler{} +} + +func (h *pluginHandler) LoadPlugin(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) + w.Write([]byte("load plugin is disabled, please `PLUGIN=1 $(MAKE) pd-server` first")) +} + +func (h *pluginHandler) UnloadPlugin(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) + w.Write([]byte("unload plugin is disabled, please `PLUGIN=1 $(MAKE) pd-server` first")) +} diff --git a/server/api/server_test.go b/server/api/server_test.go index 8693b4b87ca8..bdf929b17b3e 100644 --- a/server/api/server_test.go +++ b/server/api/server_test.go @@ -16,7 +16,9 @@ package api import ( "context" + "fmt" "net/http" + "net/http/httptest" "sort" "sync" "testing" @@ -211,3 +213,24 @@ func (suite *serviceTestSuite) TestServiceLabels() { apiutil.NewAccessPath("/pd/api/v1/metric/query", http.MethodGet)) suite.Equal("QueryMetric", serviceLabel) } + +func (suite *adminTestSuite) TestCleanPath() { + re := suite.Require() + // transfer path to /config + url := fmt.Sprintf("%s/admin/persist-file/../../config", suite.urlPrefix) + cfg := &config.Config{} + err := testutil.ReadGetJSON(re, testDialClient, url, cfg) + suite.NoError(err) + + // handled by router + response := httptest.NewRecorder() + r, _, _ := NewHandler(context.Background(), suite.svr) + request, err := http.NewRequest(http.MethodGet, url, nil) + re.NoError(err) + r.ServeHTTP(response, request) + // handled by `cleanPath` which is in `mux.ServeHTTP` + result := response.Result() + defer result.Body.Close() + re.NotNil(result.Header["Location"]) + re.Contains(result.Header["Location"][0], "/pd/api/v1/config") +} diff --git a/server/handler.go b/server/handler.go index a871b19f447f..f99b01abf99e 100644 --- a/server/handler.go +++ b/server/handler.go @@ -21,6 +21,7 @@ import ( "fmt" "net/http" "path" + "path/filepath" "strconv" "strings" "time" @@ -967,6 +968,13 @@ func (h *Handler) PluginLoad(pluginPath string) error { c := cluster.GetCoordinator() ch := make(chan string) h.pluginChMap[pluginPath] = ch + + // make sure path is in data dir + filePath, err := filepath.Abs(pluginPath) + if err != nil || !isPathInDirectory(filePath, h.s.GetConfig().DataDir) { + return errs.ErrFilePathAbs.Wrap(err).FastGenWithCause() + } + c.LoadPlugin(pluginPath, ch) return nil } diff --git a/server/replication/replication_mode.go b/server/replication/replication_mode.go index d276bd8ec185..4d54365d2901 100644 --- a/server/replication/replication_mode.go +++ b/server/replication/replication_mode.go @@ -60,7 +60,8 @@ type FileReplicater interface { ReplicateFileToMember(ctx context.Context, member *pdpb.Member, name string, data []byte) error } -const drStatusFile = "DR_STATE" +// DrStatusFile is the file name that stores the dr status. +const DrStatusFile = "DR_STATE" const persistFileTimeout = time.Second * 3 // ModeManager is used to control how raft logs are synchronized between @@ -483,7 +484,7 @@ func (m *ModeManager) tickReplicateStatus() { stateID, ok := m.replicateState.Load(member.GetMemberId()) if !ok || stateID.(uint64) != state.StateID { ctx, cancel := context.WithTimeout(context.Background(), persistFileTimeout) - err := m.fileReplicater.ReplicateFileToMember(ctx, member, drStatusFile, data) + err := m.fileReplicater.ReplicateFileToMember(ctx, member, DrStatusFile, data) if err != nil { log.Warn("failed to switch state", zap.String("replicate-mode", modeDRAutoSync), zap.String("new-state", state.State), errs.ZapError(err)) } else { diff --git a/server/server.go b/server/server.go index 37d0846876c3..a39541bded53 100644 --- a/server/server.go +++ b/server/server.go @@ -47,6 +47,18 @@ import ( "github.com/tikv/pd/pkg/jsonutil" "github.com/tikv/pd/pkg/logutil" "github.com/tikv/pd/pkg/ratelimit" +<<<<<<< HEAD +======= + "github.com/tikv/pd/pkg/replication" + sc "github.com/tikv/pd/pkg/schedule/config" + "github.com/tikv/pd/pkg/schedule/hbstream" + "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/storage" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/syncer" +>>>>>>> a21fd58d9 (security: disable plugin in default and persist file in specified dir (#7087)) "github.com/tikv/pd/pkg/systimemon" "github.com/tikv/pd/pkg/tsoutil" "github.com/tikv/pd/pkg/typeutil" @@ -1678,8 +1690,15 @@ func (s *Server) ReplicateFileToMember(ctx context.Context, member *pdpb.Member, // PersistFile saves a file in DataDir. func (s *Server) PersistFile(name string, data []byte) error { + if name != replication.DrStatusFile { + return errors.New("Invalid file name") + } log.Info("persist file", zap.String("name", name), zap.Binary("data", data)) - return os.WriteFile(filepath.Join(s.GetConfig().DataDir, name), data, 0644) // #nosec + path := filepath.Join(s.GetConfig().DataDir, name) + if !isPathInDirectory(path, s.GetConfig().DataDir) { + return errors.New("Invalid file path") + } + return os.WriteFile(path, data, 0644) // #nosec } // SaveTTLConfig save ttl config diff --git a/server/server_test.go b/server/server_test.go index cd7cf89ab971..d0b4bdbe465a 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "net/http" + "path/filepath" "testing" "github.com/stretchr/testify/suite" @@ -337,3 +338,35 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderBoth() { bodyString := string(bodyBytes) suite.Equal("Hello World\n", bodyString) } +<<<<<<< HEAD +======= + +func TestAPIService(t *testing.T) { + re := require.New(t) + + cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(re)) + defer testutil.CleanServer(cfg.DataDir) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + mockHandler := CreateMockHandler(re, "127.0.0.1") + svr, err := CreateServer(ctx, cfg, []string{utils.APIServiceName}, mockHandler) + re.NoError(err) + defer svr.Close() + err = svr.Run() + re.NoError(err) + MustWaitLeader(re, []*Server{svr}) + re.True(svr.IsAPIServiceMode()) +} + +func TestIsPathInDirectory(t *testing.T) { + re := require.New(t) + fileName := "test" + directory := "/root/project" + path := filepath.Join(directory, fileName) + re.True(isPathInDirectory(path, directory)) + + fileName = "../../test" + path = filepath.Join(directory, fileName) + re.False(isPathInDirectory(path, directory)) +} +>>>>>>> a21fd58d9 (security: disable plugin in default and persist file in specified dir (#7087)) diff --git a/server/util.go b/server/util.go index 53f4be1c666b..d68dd3758293 100644 --- a/server/util.go +++ b/server/util.go @@ -16,9 +16,15 @@ package server import ( "context" +<<<<<<< HEAD "fmt" "math/rand" "time" +======= + "net/http" + "path/filepath" + "strings" +>>>>>>> a21fd58d9 (security: disable plugin in default and persist file in specified dir (#7087)) "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/pdpb" @@ -157,3 +163,62 @@ func checkBootstrapRequest(clusterID uint64, req *pdpb.BootstrapRequest) error { return nil } +<<<<<<< HEAD +======= + +func combineBuilderServerHTTPService(ctx context.Context, svr *Server, serviceBuilders ...HandlerBuilder) (map[string]http.Handler, error) { + userHandlers := make(map[string]http.Handler) + registerMap := make(map[string]http.Handler) + + apiService := negroni.New() + recovery := negroni.NewRecovery() + apiService.Use(recovery) + router := mux.NewRouter() + + for _, build := range serviceBuilders { + handler, info, err := build(ctx, svr) + if err != nil { + return nil, err + } + if !info.IsCore && len(info.PathPrefix) == 0 && (len(info.Name) == 0 || len(info.Version) == 0) { + return nil, errs.ErrAPIInformationInvalid.FastGenByArgs(info.Name, info.Version) + } + + if err := apiutil.RegisterUserDefinedHandlers(registerMap, &info, handler); err != nil { + return nil, err + } + } + + // Combine the pd service to the router. the extension service will be added to the userHandlers. + for pathPrefix, handler := range registerMap { + if strings.Contains(pathPrefix, apiutil.CorePath) || strings.Contains(pathPrefix, apiutil.ExtensionsPath) { + router.PathPrefix(pathPrefix).Handler(handler) + if pathPrefix == apiutil.CorePath { + // Deprecated + router.Path("/pd/health").Handler(handler) + // Deprecated + router.Path("/pd/ping").Handler(handler) + } + } else { + userHandlers[pathPrefix] = handler + } + } + apiService.UseHandler(router) + userHandlers[pdAPIPrefix] = apiService + return userHandlers, nil +} + +func isPathInDirectory(path, directory string) bool { + absPath, err := filepath.Abs(path) + if err != nil { + return false + } + + absDir, err := filepath.Abs(directory) + if err != nil { + return false + } + + return strings.HasPrefix(absPath, absDir) +} +>>>>>>> a21fd58d9 (security: disable plugin in default and persist file in specified dir (#7087))