lightning: better estimate of the max-open-files (#1414)

pkg/lightning/backend/local/local_unix.go

@@ -26,8 +26,8 @@ import (
 )
 
 const (
-	// mininum max open files value
-	minRLimit = 1024
+	// maximum max open files value
+	maxRLimit = 1000000
 )
 
 func GetSystemRLimit() (uint64, error) {
@@ -39,9 +39,15 @@ func GetSystemRLimit() (uint64, error) {
 // VerifyRLimit checks whether the open-file limit is large enough.
 // In Local-backend, we need to read and write a lot of L0 SST files, so we need
 // to check system max open files limit.
+<<<<<<< HEAD
 func VerifyRLimit(estimateMaxFiles uint64) error {
 	if estimateMaxFiles < minRLimit {
 		estimateMaxFiles = minRLimit
+=======
+func VerifyRLimit(estimateMaxFiles Rlim_t) error {
+	if estimateMaxFiles > maxRLimit {
+		estimateMaxFiles = maxRLimit
+>>>>>>> 92fc8b0e (lightning: better estimate of the max-open-files (#1414))
 	}
 	var rLimit syscall.Rlimit
 	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)

pkg/lightning/lightning.go

@@ -672,7 +672,14 @@ func checkSystemRequirement(cfg *config.Config, dbsMeta []*mydump.MDDatabaseMeta
 
 		// region-concurrency: number of LocalWriters writing SST files.
 		// 2*totalSize/memCacheSize: number of Pebble MemCache files.
+<<<<<<< HEAD
 		estimateMaxFiles := uint64(cfg.App.RegionConcurrency) + uint64(topNTotalSize)/uint64(cfg.TikvImporter.EngineMemCacheSize)*2
+=======
+		maxDBFiles := topNTotalSize / int64(cfg.TikvImporter.LocalWriterMemCacheSize) * 2
+		// the pebble db and all import routine need upto maxDBFiles fds for read and write.
+		maxOpenDBFiles := maxDBFiles * (1 + int64(cfg.TikvImporter.RangeConcurrency))
+		estimateMaxFiles := local.Rlim_t(cfg.App.RegionConcurrency) + local.Rlim_t(maxOpenDBFiles)
+>>>>>>> 92fc8b0e (lightning: better estimate of the max-open-files (#1414))
 		if err := local.VerifyRLimit(estimateMaxFiles); err != nil {
 			return err
 		}

pkg/lightning/lightning_test.go

@@ -437,7 +437,8 @@ func (s *lightningServerSuite) TestCheckSystemRequirement(c *C) {
 	cfg.App.CheckRequirements = true
 	cfg.App.TableConcurrency = 4
 	cfg.TikvImporter.Backend = config.BackendLocal
-	cfg.TikvImporter.EngineMemCacheSize = 512 * units.MiB
+	cfg.TikvImporter.LocalWriterMemCacheSize = 128 * units.MiB
+	cfg.TikvImporter.RangeConcurrency = 16
 
 	dbMetas := []*mydump.MDDatabaseMeta{
 		{
@@ -476,22 +477,22 @@ func (s *lightningServerSuite) TestCheckSystemRequirement(c *C) {
 	}
 
 	// with max open files 1024, the max table size will be: 65536MB
-	err := failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue", "return(2049)")
+	err := failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue", "return(139439)")
 	c.Assert(err, IsNil)
 	err = failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/local/SetRlimitError", "return(true)")
 	c.Assert(err, IsNil)
 	defer func() {
 		_ = failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/local/SetRlimitError")
 	}()
-	// with this dbMetas, the estimated fds will be 2050, so should return error
+	// with this dbMetas, the estimated fds will be 139440, so should return error
 	err = checkSystemRequirement(cfg, dbMetas)
 	c.Assert(err, NotNil)
 
 	err = failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue")
 	c.Assert(err, IsNil)
 
 	// the min rlimit should be bigger than the default min value (16384)
-	err = failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue", "return(8200)")
+	err = failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue", "return(139440)")
 	defer func() {
 		_ = failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue")
 	}()