From 3a825efb91fd9f6645d06291fa6eded1f74d9119 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 10 May 2021 13:50:09 +0200 Subject: [PATCH] etcdserver: Implement running defrag if freeable space will exceed privided threshold --- server/config/config.go | 4 ++++ server/embed/config.go | 3 +++ server/embed/etcd.go | 1 + server/etcdmain/config.go | 1 + server/etcdmain/help.go | 2 ++ server/etcdserver/server.go | 26 ++++++++++++++++++++++++++ tests/e2e/etcd_config_test.go | 15 +++++++++++++++ 7 files changed, 52 insertions(+) diff --git a/server/config/config.go b/server/config/config.go index cbf91857b00c..c74ef97cbde3 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -188,6 +188,10 @@ type ServerConfig struct { // ExperimentalTxnModeWriteWithSharedBuffer enable write transaction to use // a shared buffer in its readonly check operations. ExperimentalTxnModeWriteWithSharedBuffer bool `json:"experimental-txn-mode-write-with-shared-buffer"` + + // ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to + // consider running defrag during bootstrap. + ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"` } // VerifyBootstrap sanity-checks the initial config for bootstrap case diff --git a/server/embed/config.go b/server/embed/config.go index dae9e7c07a58..8dae7a506878 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -314,6 +314,9 @@ type Config struct { // ExperimentalWarningApplyDuration is the time duration after which a warning is generated if applying request // takes more time than this value. ExperimentalWarningApplyDuration time.Duration `json:"experimental-warning-apply-duration"` + // ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to + // consider running defrag during bootstrap. + ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"` // ForceNewCluster starts a new cluster even if previously started; unsafe. ForceNewCluster bool `json:"force-new-cluster"` diff --git a/server/embed/etcd.go b/server/embed/etcd.go index d438a897a5c7..8ac8cf33f2b5 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -225,6 +225,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { WarningApplyDuration: cfg.ExperimentalWarningApplyDuration, ExperimentalMemoryMlock: cfg.ExperimentalMemoryMlock, ExperimentalTxnModeWriteWithSharedBuffer: cfg.ExperimentalTxnModeWriteWithSharedBuffer, + ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes, } if srvcfg.ExperimentalEnableDistributedTracing { diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index 15c35c4097cc..51f12bb486f6 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -276,6 +276,7 @@ func newConfig() *config { fs.DurationVar(&cfg.ec.ExperimentalWarningApplyDuration, "experimental-warning-apply-duration", cfg.ec.ExperimentalWarningApplyDuration, "Time duration after which a warning is generated if request takes more time.") fs.BoolVar(&cfg.ec.ExperimentalMemoryMlock, "experimental-memory-mlock", cfg.ec.ExperimentalMemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") fs.BoolVar(&cfg.ec.ExperimentalTxnModeWriteWithSharedBuffer, "experimental-txn-mode-write-with-shared-buffer", true, "Enable the write transaction to use a shared buffer in its readonly check operations.") + fs.UintVar(&cfg.ec.ExperimentalBootstrapDefragThresholdMegabytes, "experimental-bootstrap-defrag-threshold-megabytes", 0, "Enable the defrag during etcd server bootstrap on condition that it will free at least the provided threshold of disk space.") // unsafe fs.BoolVar(&cfg.ec.UnsafeNoFsync, "unsafe-no-fsync", false, "Disables fsync, unsafe, will cause data loss.") diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index c18df94d5de8..b13f1dea7592 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -236,6 +236,8 @@ Experimental feature: Warning is generated if requests take more than this duration. --experimental-txn-mode-write-with-shared-buffer 'true' Enable the write transaction to use a shared buffer in its readonly check operations. + --experimental-bootstrap-defrag-threshold-megabytes + Enable the defrag during etcd server bootstrap on condition that it will free at least the provided threshold of disk space. Unsafe feature: --force-new-cluster 'false' diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index e5c3dc1a2d5b..ab62888c500c 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -362,6 +362,13 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { ci.SetBackend(be) cindex.CreateMetaBucket(be.BatchTx()) + if cfg.ExperimentalBootstrapDefragThresholdMegabytes != 0 { + err := maybeDefragBackend(cfg, be) + if err != nil { + return nil, err + } + } + defer func() { if err != nil { be.Close() @@ -2580,3 +2587,22 @@ func (s *EtcdServer) IsMemberExist(id types.ID) bool { func (s *EtcdServer) raftStatus() raft.Status { return s.r.Node.Status() } + +func maybeDefragBackend(cfg config.ServerConfig, be backend.Backend) error { + size := be.Size() + sizeInUse := be.SizeInUse() + freeableMemory := uint(size - sizeInUse) + thresholdBytes := cfg.ExperimentalBootstrapDefragThresholdMegabytes * 1024 * 1024 + if freeableMemory < thresholdBytes { + cfg.Logger.Info("Skipping defragmentation", + zap.Int64("current-db-size-bytes", size), + zap.String("current-db-size", humanize.Bytes(uint64(size))), + zap.Int64("current-db-size-in-use-bytes", sizeInUse), + zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse))), + zap.Uint("experimental-bootstrap-defrag-threshold-bytes", thresholdBytes), + zap.String("experimental-bootstrap-defrag-threshold", humanize.Bytes(uint64(thresholdBytes))), + ) + return nil + } + return be.Defrag() +} diff --git a/tests/e2e/etcd_config_test.go b/tests/e2e/etcd_config_test.go index 0a0b58032d4e..ef39a52e5888 100644 --- a/tests/e2e/etcd_config_test.go +++ b/tests/e2e/etcd_config_test.go @@ -319,3 +319,18 @@ func TestGrpcproxyAndCommonName(t *testing.T) { t.Fatal(err) } } + +func TestBootstrapDefragFlag(t *testing.T) { + skipInShortMode(t) + + proc, err := spawnCmd([]string{binDir + "/etcd", "--experimental-bootstrap-defrag-threshold-megabytes", "1000"}) + if err != nil { + t.Fatal(err) + } + if err = waitReadyExpectProc(proc, []string{"Skipping defragmentation"}); err != nil { + t.Fatal(err) + } + if err = proc.Stop(); err != nil { + t.Fatal(err) + } +}