From 6abc349dc97d7b68bf40e79f0c7510cc52b1f38e Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Thu, 9 May 2024 21:33:55 -0400 Subject: [PATCH 1/2] server: Implement WithMmapSize option for backend config Accept a third argument for NewDefaultBackend for overrides to the BackendConfig. Add a new function, WithMmapSize, which modifies the backend config to provide a custom InitiamMmapSize. Backports commit: d69adf45f9e95661966204460f019b8164aaef12 / PR: #17277 Signed-off-by: Ivan Valdes --- server/mvcc/backend/backend.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/server/mvcc/backend/backend.go b/server/mvcc/backend/backend.go index f156ae9948f..a986921e225 100644 --- a/server/mvcc/backend/backend.go +++ b/server/mvcc/backend/backend.go @@ -151,6 +151,8 @@ type BackendConfig struct { Hooks Hooks } +type BackendConfigOption func(*BackendConfig) + func DefaultBackendConfig() BackendConfig { return BackendConfig{ BatchInterval: defaultBatchInterval, @@ -163,9 +165,19 @@ func New(bcfg BackendConfig) Backend { return newBackend(bcfg) } -func NewDefaultBackend(path string) Backend { +func WithMmapSize(size uint64) BackendConfigOption { + return func(bcfg *BackendConfig) { + bcfg.MmapSize = size + } +} + +func NewDefaultBackend(path string, opts ...BackendConfigOption) Backend { bcfg := DefaultBackendConfig() bcfg.Path = path + for _, opt := range opts { + opt(&bcfg) + } + return newBackend(bcfg) } From 9e1dadd74850ac70b49df06785d12133e429f5c9 Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Thu, 9 May 2024 21:37:10 -0400 Subject: [PATCH 2/2] etcdutl: Fix snapshot restore memory alloc issue When running the snapshot command, allow receiving an initial memory map allocation for the database, avoiding future memory allocation issues. Backports commit: be2883321240340479f9e334bc4a3b2959ad7639 / PR: #17277 Co-authored-by: Fatih USTA Signed-off-by: Ivan Valdes --- etcdctl/ctlv3/command/snapshot_command.go | 4 +++- etcdutl/etcdutl/snapshot_command.go | 7 ++++++- etcdutl/snapshot/v3_snapshot.go | 15 +++++++++++---- server/mvcc/backend/backend.go | 6 +++--- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/etcdctl/ctlv3/command/snapshot_command.go b/etcdctl/ctlv3/command/snapshot_command.go index c017dc36db3..eb0227d8181 100644 --- a/etcdctl/ctlv3/command/snapshot_command.go +++ b/etcdctl/ctlv3/command/snapshot_command.go @@ -42,6 +42,7 @@ var ( skipHashCheck bool markCompacted bool revisionBump uint64 + initialMmapSize uint64 ) // NewSnapshotCommand returns the cobra command for "snapshot". @@ -93,6 +94,7 @@ func NewSnapshotRestoreCommand() *cobra.Command { cmd.Flags().BoolVar(&skipHashCheck, "skip-hash-check", false, "Ignore snapshot integrity hash value (required if copied from data directory)") cmd.Flags().Uint64Var(&revisionBump, "bump-revision", 0, "How much to increase the latest revision after restore") cmd.Flags().BoolVar(&markCompacted, "mark-compacted", false, "Mark the latest revision after restore as the point of scheduled compaction (required if --bump-revision > 0, disallowed otherwise)") + cmd.Flags().Uint64Var(&initialMmapSize, "initial-memory-map-size", initialMmapSize, "Initial memory map size of the database in bytes. It uses the default value if not defined or defined to 0") return cmd } @@ -131,7 +133,7 @@ func snapshotStatusCommandFunc(cmd *cobra.Command, args []string) { func snapshotRestoreCommandFunc(cmd *cobra.Command, args []string) { fmt.Fprintf(os.Stderr, "Deprecated: Use `etcdutl snapshot restore` instead.\n\n") etcdutl.SnapshotRestoreCommandFunc(restoreCluster, restoreClusterToken, restoreDataDir, restoreWalDir, - restorePeerURLs, restoreName, skipHashCheck, revisionBump, markCompacted, args) + restorePeerURLs, restoreName, skipHashCheck, initialMmapSize, revisionBump, markCompacted, args) } func initialClusterFromName(name string) string { diff --git a/etcdutl/etcdutl/snapshot_command.go b/etcdutl/etcdutl/snapshot_command.go index 655b48fdef3..a7d9397bccc 100644 --- a/etcdutl/etcdutl/snapshot_command.go +++ b/etcdutl/etcdutl/snapshot_command.go @@ -21,6 +21,7 @@ import ( "go.etcd.io/etcd/etcdutl/v3/snapshot" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.etcd.io/etcd/server/v3/datadir" + "go.etcd.io/etcd/server/v3/mvcc/backend" "github.com/spf13/cobra" ) @@ -38,6 +39,7 @@ var ( restorePeerURLs string restoreName string skipHashCheck bool + initialMmapSize = backend.InitialMmapSize markCompacted bool revisionBump uint64 ) @@ -93,6 +95,7 @@ func NewSnapshotRestoreCommand() *cobra.Command { cmd.Flags().StringVar(&restorePeerURLs, "initial-advertise-peer-urls", defaultInitialAdvertisePeerURLs, "List of this member's peer URLs to advertise to the rest of the cluster") cmd.Flags().StringVar(&restoreName, "name", defaultName, "Human-readable name for this member") cmd.Flags().BoolVar(&skipHashCheck, "skip-hash-check", false, "Ignore snapshot integrity hash value (required if copied from data directory)") + cmd.Flags().Uint64Var(&initialMmapSize, "initial-memory-map-size", initialMmapSize, "Initial memory map size of the database in bytes. It uses the default value if not defined or defined to 0") cmd.Flags().Uint64Var(&revisionBump, "bump-revision", 0, "How much to increase the latest revision after restore") cmd.Flags().BoolVar(&markCompacted, "mark-compacted", false, "Mark the latest revision after restore as the point of scheduled compaction (required if --bump-revision > 0, disallowed otherwise)") @@ -119,7 +122,7 @@ func SnapshotStatusCommandFunc(cmd *cobra.Command, args []string) { func snapshotRestoreCommandFunc(_ *cobra.Command, args []string) { SnapshotRestoreCommandFunc(restoreCluster, restoreClusterToken, restoreDataDir, restoreWalDir, - restorePeerURLs, restoreName, skipHashCheck, revisionBump, markCompacted, args) + restorePeerURLs, restoreName, skipHashCheck, initialMmapSize, revisionBump, markCompacted, args) } func SnapshotRestoreCommandFunc(restoreCluster string, @@ -129,6 +132,7 @@ func SnapshotRestoreCommandFunc(restoreCluster string, restorePeerURLs string, restoreName string, skipHashCheck bool, + initialMmapSize uint64, revisionBump uint64, markCompacted bool, args []string) { @@ -164,6 +168,7 @@ func SnapshotRestoreCommandFunc(restoreCluster string, InitialCluster: restoreCluster, InitialClusterToken: restoreClusterToken, SkipHashCheck: skipHashCheck, + InitialMmapSize: initialMmapSize, RevisionBump: revisionBump, MarkCompacted: markCompacted, }); err != nil { diff --git a/etcdutl/snapshot/v3_snapshot.go b/etcdutl/snapshot/v3_snapshot.go index bd0d4a7315d..882a29dc49c 100644 --- a/etcdutl/snapshot/v3_snapshot.go +++ b/etcdutl/snapshot/v3_snapshot.go @@ -83,7 +83,8 @@ type v3Manager struct { snapDir string cl *membership.RaftCluster - skipHashCheck bool + skipHashCheck bool + initialMmapSize uint64 } // hasChecksum returns "true" if the file size "n" @@ -197,6 +198,9 @@ type RestoreConfig struct { // (required if copied from data directory). SkipHashCheck bool + // InitialMmapSize is the database initial memory map size. + InitialMmapSize uint64 + // RevisionBump is the amount to increase the latest revision after restore, // to allow administrators to trick clients into thinking that revision never decreased. // If 0, revision bumping is skipped. @@ -256,6 +260,7 @@ func (s *v3Manager) Restore(cfg RestoreConfig) error { s.walDir = walDir s.snapDir = filepath.Join(dataDir, "member", "snap") s.skipHashCheck = cfg.SkipHashCheck + s.initialMmapSize = cfg.InitialMmapSize s.lg.Info( "restoring snapshot", @@ -263,6 +268,7 @@ func (s *v3Manager) Restore(cfg RestoreConfig) error { zap.String("wal-dir", s.walDir), zap.String("data-dir", dataDir), zap.String("snap-dir", s.snapDir), + zap.Uint64("initial-memory-map-size", s.initialMmapSize), ) if err = s.saveDB(); err != nil { @@ -290,6 +296,7 @@ func (s *v3Manager) Restore(cfg RestoreConfig) error { zap.String("wal-dir", s.walDir), zap.String("data-dir", dataDir), zap.String("snap-dir", s.snapDir), + zap.Uint64("initial-memory-map-size", s.initialMmapSize), ) return verify.VerifyIfEnabled(verify.Config{ @@ -310,7 +317,7 @@ func (s *v3Manager) saveDB() error { return err } - be := backend.NewDefaultBackend(s.outDbPath()) + be := backend.NewDefaultBackend(s.outDbPath(), backend.WithMmapSize(s.initialMmapSize)) defer be.Close() err = membership.TrimMembershipFromBackend(s.lg, be) @@ -324,7 +331,7 @@ func (s *v3Manager) saveDB() error { // modifyLatestRevision can increase the latest revision by the given amount and sets the scheduled compaction // to that revision so that the server will consider this revision compacted. func (s *v3Manager) modifyLatestRevision(bumpAmount uint64) error { - be := backend.NewDefaultBackend(s.outDbPath()) + be := backend.NewDefaultBackend(s.outDbPath(), backend.WithMmapSize(s.initialMmapSize)) defer func() { be.ForceCommit() be.Close() @@ -467,7 +474,7 @@ func (s *v3Manager) saveWALAndSnap() (*raftpb.HardState, error) { // add members again to persist them to the store we create. st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix) s.cl.SetStore(st) - be := backend.NewDefaultBackend(s.outDbPath()) + be := backend.NewDefaultBackend(s.outDbPath(), backend.WithMmapSize(s.initialMmapSize)) defer be.Close() s.cl.SetBackend(be) for _, m := range s.cl.Members() { diff --git a/server/mvcc/backend/backend.go b/server/mvcc/backend/backend.go index a986921e225..7d77da12fd6 100644 --- a/server/mvcc/backend/backend.go +++ b/server/mvcc/backend/backend.go @@ -36,10 +36,10 @@ var ( defragLimit = 10000 - // initialMmapSize is the initial size of the mmapped region. Setting this larger than + // InitialMmapSize is the initial size of the mmapped region. Setting this larger than // the potential max db size can prevent writer from blocking reader. // This only works for linux. - initialMmapSize = uint64(10 * 1024 * 1024 * 1024) + InitialMmapSize = uint64(10 * 1024 * 1024 * 1024) // minSnapshotWarningTimeout is the minimum threshold to trigger a long running snapshot warning. minSnapshotWarningTimeout = 30 * time.Second @@ -157,7 +157,7 @@ func DefaultBackendConfig() BackendConfig { return BackendConfig{ BatchInterval: defaultBatchInterval, BatchLimit: defaultBatchLimit, - MmapSize: initialMmapSize, + MmapSize: InitialMmapSize, } }