Merge pull request #15 from 0chain/debug/mpt-save-slow

Tune MPT rocksdb to mitigate write stall
0chain · Sep 24, 2023 · d463636 · d463636
2 parents 1e2e9da + fef500a
commit d463636
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 2 deletions.
diff --git a/core/util/merkle_patricia_trie.go b/core/util/merkle_patricia_trie.go
@@ -240,6 +240,7 @@ func (mpt *MerklePatriciaTrie) SaveChanges(ctx context.Context, ndb NodeDB, incl
 		if err != nil {
 			logging.Logger.Error("MPT save changes failed",
 				zap.Any("version", mpt.Version),
+				zap.Int("changes", len(cc.GetChanges())),
 				zap.Error(err))
 			errC <- err
 		}
@@ -249,10 +250,13 @@ func (mpt *MerklePatriciaTrie) SaveChanges(ctx context.Context, ndb NodeDB, incl
 	case <-ctx.Done():
 		Logger.Debug("MPT save changes timeout",
 			zap.Any("duration", time.Since(ts)),
+			zap.Int("changes", len(cc.GetChanges())),
 			zap.Error(ctx.Err()))
 		return ctx.Err()
 	case err := <-errC:
-		Logger.Debug("MPT save changes failed", zap.Error(err))
+		Logger.Debug("MPT save changes failed",
+			zap.Int("changes", len(cc.GetChanges())),
+			zap.Error(err))
 		return err
 	case <-doneC:
 	}

diff --git a/core/util/mpt_node_change.go b/core/util/mpt_node_change.go
@@ -144,10 +144,13 @@ func (cc *ChangeCollector) UpdateChanges(ndb NodeDB, origin Sequence, includeDel
 		keys[idx] = nodes[idx].GetHashBytes()
 		idx++
 	}
+
+	logging.Logger.Debug("MPT - update changes", zap.Int("changes", len(keys)))
 	err := ndb.MultiPutNode(keys, nodes)
 	if err != nil {
 		return err
 	}
+	logging.Logger.Debug("MPT - update changes done", zap.Int("changes", len(keys)))
 	if includeDeletes {
 		for _, d := range cc.Deletes {
 			err := ndb.DeleteNode(d.GetHashBytes())

diff --git a/core/util/mpt_pnodedb.go b/core/util/mpt_pnodedb.go
@@ -44,6 +44,7 @@ func newDefaultCFOptions(logDir string) *grocksdb.Options {
 	opts := grocksdb.NewDefaultOptions()
 	opts.SetCreateIfMissing(true)
 	opts.SetCompression(PNodeDBCompression)
+	opts.OptimizeUniversalStyleCompaction(64 * 1024 * 1024)
 	if sstType == SSTTypePlainTable {
 		opts.SetAllowMmapReads(true)
 		opts.SetPrefixExtractor(grocksdb.NewFixedPrefixTransform(6))
@@ -52,11 +53,14 @@ func newDefaultCFOptions(logDir string) *grocksdb.Options {
 		opts.OptimizeForPointLookup(64)
 		opts.SetAllowMmapReads(true)
 		opts.SetPrefixExtractor(grocksdb.NewFixedPrefixTransform(6))
+		opts.SetMaxBackgroundJobs(4)               // default was 2, double to 4
+		opts.SetMaxWriteBufferNumber(4)            // default was 2, double to 4
+		opts.SetWriteBufferSize(128 * 1024 * 1024) // default was 64M, double to 128M
+		opts.SetMinWriteBufferNumberToMerge(2)     // default was 1, double to 2
 	}
 	opts.IncreaseParallelism(2) // pruning and saving happen in parallel
 	opts.SetDbLogDir(logDir)
 	opts.EnableStatistics()
-	opts.OptimizeUniversalStyleCompaction(64 * 1024 * 1024)
 
 	return opts
 }
@@ -69,6 +73,11 @@ func newDeadNodesCFOptions() *grocksdb.Options {
 	opts.SetBlockBasedTableFactory(bbto)
 	opts.SetCreateIfMissing(true)
 	opts.SetCompression(PNodeDBCompression)
+
+	opts.SetMaxBackgroundJobs(4)               // default was 2, double to 4
+	opts.SetMaxWriteBufferNumber(4)            // default was 2, double to 4
+	opts.SetWriteBufferSize(128 * 1024 * 1024) // default was 64M, double to 128M
+	opts.SetMinWriteBufferNumberToMerge(2)     // default was 1, double to 2
 	return opts
 }