diff --git a/backup_test.go b/backup_test.go index 2c4115721..31b7cd95f 100644 --- a/backup_test.go +++ b/backup_test.go @@ -128,10 +128,7 @@ func TestBackupRestore2(t *testing.T) { s2Path := filepath.Join(tmpdir, "test2") s3Path := filepath.Join(tmpdir, "test3") - opts := DefaultOptions - opts.Dir = s1Path - opts.ValueDir = s1Path - db1, err := Open(opts) + db1, err := Open(DefaultOptions(s1Path)) if err != nil { t.Fatal(err) } @@ -166,10 +163,7 @@ func TestBackupRestore2(t *testing.T) { } fmt.Println("backup1 length:", backup.Len()) - opts = DefaultOptions - opts.Dir = s2Path - opts.ValueDir = s2Path - db2, err := Open(opts) + db2, err := Open(DefaultOptions(s2Path)) if err != nil { t.Fatal(err) } @@ -220,10 +214,7 @@ func TestBackupRestore2(t *testing.T) { t.Fatal(err) } fmt.Println("backup2 length:", backup.Len()) - opts = DefaultOptions - opts.Dir = s3Path - opts.ValueDir = s3Path - db3, err := Open(opts) + db3, err := Open(DefaultOptions(s3Path)) if err != nil { t.Fatal(err) } @@ -296,10 +287,7 @@ func TestBackup(t *testing.T) { } defer os.RemoveAll(tmpdir) - opts := DefaultOptions - opts.Dir = filepath.Join(tmpdir, "backup0") - opts.ValueDir = opts.Dir - db1, err := Open(opts) + db1, err := Open(DefaultOptions(filepath.Join(tmpdir, "backup0"))) if err != nil { t.Fatal(err) } @@ -344,15 +332,12 @@ func TestBackupRestore3(t *testing.T) { } defer os.RemoveAll(tmpdir) - opts := DefaultOptions N := 1000 entries := createEntries(N) // backup { - opts.Dir = filepath.Join(tmpdir, "backup1") - opts.ValueDir = opts.Dir - db1, err := Open(opts) + db1, err := Open(DefaultOptions(filepath.Join(tmpdir, "backup1"))) if err != nil { t.Fatal(err) } @@ -367,9 +352,7 @@ func TestBackupRestore3(t *testing.T) { require.True(t, bb.Len() > 0) // restore - opts.Dir = filepath.Join(tmpdir, "restore1") - opts.ValueDir = opts.Dir - db2, err := Open(opts) + db2, err := Open(DefaultOptions(filepath.Join(tmpdir, "restore1"))) if err != nil { t.Fatal(err) } @@ -407,7 +390,6 @@ func TestBackupLoadIncremental(t *testing.T) { } defer os.RemoveAll(tmpdir) - opts := DefaultOptions N := 100 entries := createEntries(N) updates := make(map[int]byte) @@ -415,9 +397,7 @@ func TestBackupLoadIncremental(t *testing.T) { // backup { - opts.Dir = filepath.Join(tmpdir, "backup2") - opts.ValueDir = opts.Dir - db1, err := Open(opts) + db1, err := Open(DefaultOptions(filepath.Join(tmpdir, "backup2"))) if err != nil { t.Fatal(err) } @@ -477,9 +457,7 @@ func TestBackupLoadIncremental(t *testing.T) { require.True(t, bb.Len() > 0) // restore - opts.Dir = filepath.Join(tmpdir, "restore2") - opts.ValueDir = opts.Dir - db2, err := Open(opts) + db2, err := Open(DefaultOptions(filepath.Join(tmpdir, "restore2"))) if err != nil { t.Fatal(err) } diff --git a/badger/cmd/backup.go b/badger/cmd/backup.go index 72d22aa36..79fa9fd49 100644 --- a/badger/cmd/backup.go +++ b/badger/cmd/backup.go @@ -51,11 +51,9 @@ func init() { func doBackup(cmd *cobra.Command, args []string) error { // Open DB - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.Truncate = truncate - db, err := badger.Open(opts) + db, err := badger.Open(badger.DefaultOptions(sstDir). + WithValueDir(vlogDir). + WithTruncate(truncate)) if err != nil { return err } diff --git a/badger/cmd/bank.go b/badger/cmd/bank.go index f866e4d84..ee68bd001 100644 --- a/badger/cmd/bank.go +++ b/badger/cmd/bank.go @@ -274,14 +274,11 @@ func compareTwo(db *badger.DB, before, after uint64) { } func runDisect(cmd *cobra.Command, args []string) error { - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.ReadOnly = true - // The total did not match up. So, let's disect the DB to find the // transction which caused the total mismatch. - db, err := badger.OpenManaged(opts) + db, err := badger.OpenManaged(badger.DefaultOptions(sstDir). + WithValueDir(vlogDir). + WithReadOnly(true)) if err != nil { return err } @@ -324,17 +321,16 @@ func runTest(cmd *cobra.Command, args []string) error { rand.Seed(time.Now().UnixNano()) // Open DB - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.MaxTableSize = 4 << 20 // Force more compactions. - opts.NumLevelZeroTables = 2 - opts.NumMemtables = 2 - // Do not GC any versions, because we need them for the disect. - opts.NumVersionsToKeep = int(math.MaxInt32) - opts.ValueThreshold = 1 // Make all values go to value log. + opts := badger.DefaultOptions(sstDir). + WithValueDir(vlogDir). + WithMaxTableSize(4 << 20). // Force more compactions. + WithNumLevelZeroTables(2). + WithNumMemtables(2). + // Do not GC any versions, because we need them for the disect.. + WithNumVersionsToKeep(int(math.MaxInt32)). + WithValueThreshold(1) // Make all values go to value log if mmap { - opts.TableLoadingMode = options.MemoryMap + opts = opts.WithTableLoadingMode(options.MemoryMap) } log.Printf("Opening DB with options: %+v\n", opts) @@ -350,13 +346,7 @@ func runTest(cmd *cobra.Command, args []string) error { dir, err := ioutil.TempDir("", "bank_subscribe") y.Check(err) - subscribeOpts := badger.DefaultOptions - subscribeOpts.Dir = dir - subscribeOpts.ValueDir = dir - subscribeOpts.SyncWrites = false - log.Printf("Opening subscribe DB with options: %+v\n", subscribeOpts) - - subscribeDB, err = badger.Open(subscribeOpts) + subscribeDB, err = badger.Open(badger.DefaultOptions(dir).WithSyncWrites(false)) if err != nil { return err } @@ -367,13 +357,7 @@ func runTest(cmd *cobra.Command, args []string) error { dir, err := ioutil.TempDir("", "bank_stream") y.Check(err) - streamOpts := badger.DefaultOptions - streamOpts.Dir = dir - streamOpts.ValueDir = dir - streamOpts.SyncWrites = false - log.Printf("Opening stream DB with options: %+v\n", streamOpts) - - tmpDb, err = badger.Open(streamOpts) + tmpDb, err = badger.Open(badger.DefaultOptions(dir).WithSyncWrites(false)) if err != nil { return err } diff --git a/badger/cmd/flatten.go b/badger/cmd/flatten.go index 4d58bccae..1882cb99c 100644 --- a/badger/cmd/flatten.go +++ b/badger/cmd/flatten.go @@ -40,13 +40,10 @@ func init() { } func flatten(cmd *cobra.Command, args []string) error { - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.Truncate = truncate - opts.NumCompactors = 0 - - db, err := badger.Open(opts) + db, err := badger.Open(badger.DefaultOptions(sstDir). + WithValueDir(vlogDir). + WithTruncate(truncate). + WithNumCompactors(0)) if err != nil { return err } diff --git a/badger/cmd/info.go b/badger/cmd/info.go index da57851d6..420d2e3b3 100644 --- a/badger/cmd/info.go +++ b/badger/cmd/info.go @@ -87,13 +87,10 @@ func handleInfo(cmd *cobra.Command, args []string) error { } // Open DB - opts := badger.DefaultOptions - opts.TableLoadingMode = options.MemoryMap - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.ReadOnly = true - - db, err := badger.Open(opts) + db, err := badger.Open(badger.DefaultOptions(sstDir). + WithValueDir(vlogDir). + WithReadOnly(true). + WithTableLoadingMode(options.MemoryMap)) if err != nil { return errors.Wrap(err, "failed to open database") } diff --git a/badger/cmd/read_bench.go b/badger/cmd/read_bench.go index b2821bd1e..b26a1fb27 100644 --- a/badger/cmd/read_bench.go +++ b/badger/cmd/read_bench.go @@ -79,13 +79,11 @@ func readBench(cmd *cobra.Command, args []string) error { y.AssertTrue(numGoroutines > 0) mode := getLoadingMode(loadingMode) - opts := badger.DefaultOptions - opts.ReadOnly = readOnly - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.TableLoadingMode = mode - opts.ValueLogLoadingMode = mode - db, err := badger.Open(opts) + db, err := badger.Open(badger.DefaultOptions(sstDir). + WithValueDir(vlogDir). + WithReadOnly(readOnly). + WithTableLoadingMode(mode). + WithValueLogLoadingMode(mode)) if err != nil { return y.Wrapf(err, "unable to open DB") } diff --git a/badger/cmd/restore.go b/badger/cmd/restore.go index 8efa7f3e3..d5ee8f5f4 100644 --- a/badger/cmd/restore.go +++ b/badger/cmd/restore.go @@ -65,10 +65,7 @@ func doRestore(cmd *cobra.Command, args []string) error { } // Open DB - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - db, err := badger.Open(opts) + db, err := badger.Open(badger.DefaultOptions(sstDir).WithValueDir(vlogDir)) if err != nil { return err } diff --git a/badger/cmd/write_bench.go b/badger/cmd/write_bench.go index 746e7d34c..d7d26aadb 100644 --- a/badger/cmd/write_bench.go +++ b/badger/cmd/write_bench.go @@ -155,15 +155,12 @@ func writeSorted(db *badger.DB, num uint64) error { } func writeBench(cmd *cobra.Command, args []string) error { - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.Truncate = truncate - opts.SyncWrites = false - opts.CompactL0OnClose = force - opts.Logger = nil - - db, err := badger.Open(opts) + db, err := badger.Open(badger.DefaultOptions(sstDir). + WithValueDir(vlogDir). + WithTruncate(truncate). + WithSyncWrites(false). + WithCompactL0OnClose(force). + WithLogger(nil)) if err != nil { return err } diff --git a/db2_test.go b/db2_test.go index 304514c24..d301fa116 100644 --- a/db2_test.go +++ b/db2_test.go @@ -196,9 +196,11 @@ func TestBigKeyValuePairs(t *testing.T) { t.Skip("Skipping test meant to be run manually.") return } - opts := DefaultOptions - opts.MaxTableSize = 1 << 20 - opts.ValueLogMaxEntries = 64 + + // Passing an empty directory since it will be filled by runBadgerTest. + opts := DefaultOptions(""). + WithMaxTableSize(1 << 20). + WithValueLogMaxEntries(64) runBadgerTest(t, &opts, func(t *testing.T, db *DB) { bigK := make([]byte, 65001) bigV := make([]byte, db.opt.ValueLogFileSize+1) @@ -285,9 +287,11 @@ func TestPushValueLogLimit(t *testing.T) { t.Skip("Skipping test meant to be run manually.") return } - opt := DefaultOptions - opt.ValueLogMaxEntries = 64 - opt.ValueLogFileSize = 2 << 30 + + // Passing an empty directory since it will be filled by runBadgerTest. + opt := DefaultOptions(""). + WithValueLogMaxEntries(64). + WithValueLogFileSize(2 << 30) runBadgerTest(t, &opt, func(t *testing.T, db *DB) { data := []byte(fmt.Sprintf("%30d", 1)) key := func(i int) string { @@ -338,10 +342,7 @@ func TestDiscardMapTooBig(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - ops := DefaultOptions - ops.Dir = dir - ops.ValueDir = dir - db, err := Open(ops) + db, err := Open(DefaultOptions(dir)) require.NoError(t, err, "error while openning db") // Add some data so that memtable flush happens on close @@ -356,7 +357,7 @@ func TestDiscardMapTooBig(t *testing.T) { require.NoError(t, db.Close()) // reopen the same DB - db, err = Open(ops) + db, err = Open(DefaultOptions(dir)) require.NoError(t, err, "error while openning db") require.NoError(t, db.Close()) } diff --git a/db_test.go b/db_test.go index 3d0c9675c..1d1110ecc 100644 --- a/db_test.go +++ b/db_test.go @@ -71,14 +71,12 @@ func (s *levelHandler) getSummary(sum *summary) { func (s *DB) validate() error { return s.lc.validate() } func getTestOptions(dir string) Options { - opt := DefaultOptions - opt.MaxTableSize = 1 << 15 // Force more compaction. - opt.LevelOneSize = 4 << 15 // Force more compaction. - opt.Dir = dir - opt.ValueDir = dir - opt.SyncWrites = false + opt := DefaultOptions(dir). + WithMaxTableSize(1 << 15). // Force more compaction. + WithLevelOneSize(4 << 15). // Force more compaction. + WithSyncWrites(false) if !*mmap { - opt.ValueLogLoadingMode = options.FileIO + return opt.WithValueLogLoadingMode(options.FileIO) } return opt } @@ -723,7 +721,7 @@ func TestIterateParallel(t *testing.T) { itr.Close() // Double close. } - opt := DefaultOptions + opt := DefaultOptions("") runBadgerTest(t, &opt, func(t *testing.T, db *DB) { var wg sync.WaitGroup var txns []*Txn @@ -767,10 +765,7 @@ func TestDeleteWithoutSyncWrite(t *testing.T) { dir, err := ioutil.TempDir("", "badger-test") require.NoError(t, err) defer os.RemoveAll(dir) - opt := DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - kv, err := Open(opt) + kv, err := Open(DefaultOptions(dir)) if err != nil { t.Error(err) t.Fail() @@ -783,7 +778,7 @@ func TestDeleteWithoutSyncWrite(t *testing.T) { kv.Close() // Reopen KV - kv, err = Open(opt) + kv, err = Open(DefaultOptions(dir)) if err != nil { t.Error(err) t.Fail() @@ -1139,13 +1134,7 @@ func TestLargeKeys(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - opts := new(Options) - *opts = DefaultOptions - opts.ValueLogFileSize = 1024 * 1024 * 1024 - opts.Dir = dir - opts.ValueDir = dir - - db, err := Open(*opts) + db, err := Open(DefaultOptions(dir).WithValueLogFileSize(1024 * 1024 * 1024)) if err != nil { t.Fatal(err) } @@ -1177,11 +1166,7 @@ func TestCreateDirs(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - opts := DefaultOptions - dir = filepath.Join(dir, "badger") - opts.Dir = dir - opts.ValueDir = dir - db, err := Open(opts) + db, err := Open(DefaultOptions(filepath.Join(dir, "badger"))) require.NoError(t, err) db.Close() _, err = os.Stat(dir) @@ -1194,11 +1179,7 @@ func TestGetSetDeadlock(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - opt := DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - opt.ValueLogFileSize = 1 << 20 - db, err := Open(opt) + db, err := Open(DefaultOptions(dir).WithValueLogFileSize(1 << 20)) require.NoError(t, err) defer db.Close() @@ -1241,11 +1222,7 @@ func TestWriteDeadlock(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - opt := DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - opt.ValueLogFileSize = 10 << 20 - db, err := Open(opt) + db, err := Open(DefaultOptions(dir).WithValueLogFileSize(10 << 20)) require.NoError(t, err) print := func(count *int) { @@ -1464,11 +1441,8 @@ func TestLSMOnly(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - opts := LSMOnlyOptions - opts.Dir = dir - opts.ValueDir = dir - - dopts := DefaultOptions + opts := LSMOnlyOptions(dir) + dopts := DefaultOptions(dir) require.NotEqual(t, dopts.ValueThreshold, opts.ValueThreshold) dopts.ValueThreshold = 1 << 16 @@ -1583,10 +1557,7 @@ func ExampleOpen() { log.Fatal(err) } defer os.RemoveAll(dir) - opts := DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - db, err := Open(opts) + db, err := Open(DefaultOptions(dir)) if err != nil { log.Fatal(err) } @@ -1642,11 +1613,7 @@ func ExampleTxn_NewIterator() { } defer os.RemoveAll(dir) - opts := DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - - db, err := Open(opts) + db, err := Open(DefaultOptions(dir)) if err != nil { log.Fatal(err) } @@ -1701,12 +1668,7 @@ func TestSyncForRace(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - opts := DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - opts.SyncWrites = false - - db, err := Open(opts) + db, err := Open(DefaultOptions(dir).WithSyncWrites(false)) require.NoError(t, err) defer db.Close() diff --git a/integration/testgc/main.go b/integration/testgc/main.go index 5fc4809cd..4afc46cd4 100644 --- a/integration/testgc/main.go +++ b/integration/testgc/main.go @@ -102,14 +102,10 @@ func main() { dir := "/mnt/drive/badgertest" os.RemoveAll(dir) - opts := badger.DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - opts.TableLoadingMode = options.MemoryMap - opts.ValueLogLoadingMode = options.FileIO - opts.SyncWrites = false - - db, err := badger.Open(opts) + db, err := badger.Open(badger.DefaultOptions(dir). + WithTableLoadingMode(options.MemoryMap). + WithValueLogLoadingMode(options.FileIO). + WithSyncWrites(false)) if err != nil { log.Fatal(err) } diff --git a/manifest_test.go b/manifest_test.go index 12ddeef93..c6dcf7d93 100644 --- a/manifest_test.go +++ b/manifest_test.go @@ -164,10 +164,7 @@ func TestOverlappingKeyRangeError(t *testing.T) { dir, err := ioutil.TempDir("", "badger-test") require.NoError(t, err) defer os.RemoveAll(dir) - opt := DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - kv, err := Open(opt) + kv, err := Open(DefaultOptions(dir)) require.NoError(t, err) lh0 := newLevelHandler(kv, 0) diff --git a/options.go b/options.go index 3b5b1956c..4d25a8772 100644 --- a/options.go +++ b/options.go @@ -20,92 +20,48 @@ import ( "github.com/dgraph-io/badger/v2/options" ) -// NOTE: Keep the comments in the following to 75 chars width, so they -// format nicely in godoc. +// Note: If you add a new option X make sure you also add a WithX method on Options. // Options are params for creating DB object. // // This package provides DefaultOptions which contains options that should // work for most applications. Consider using that as a starting point before // customizing it for your own needs. +// +// Each option X is documented on the WithX method. type Options struct { - // 1. Mandatory flags - // ------------------- - // Directory to store the data in. If it doesn't exist, Badger will - // try to create it for you. - Dir string - // Directory to store the value log in. Can be the same as Dir. If it - // doesn't exist, Badger will try to create it for you. - ValueDir string + // Required options. - // 2. Frequently modified flags - // ----------------------------- - // Sync all writes to disk. Setting this to false would achieve better - // performance, but may cause data to be lost. - SyncWrites bool + Dir string + ValueDir string - // How should LSM tree be accessed. - TableLoadingMode options.FileLoadingMode + // Usually modified options. - // How should value log be accessed. + SyncWrites bool + TableLoadingMode options.FileLoadingMode ValueLogLoadingMode options.FileLoadingMode + NumVersionsToKeep int + ReadOnly bool + Truncate bool + Logger Logger - // How many versions to keep per key. - NumVersionsToKeep int - - // Open the DB as read-only. With this set, multiple processes can - // open the same Badger DB. Note: if the DB being opened had crashed - // before and has vlog data to be replayed, ReadOnly will cause Open - // to fail with an appropriate message. - ReadOnly bool - - // Truncate value log to delete corrupt data, if any. Would not truncate if ReadOnly is set. - Truncate bool - - // DB-specific logger which will override the global logger. - Logger Logger - - // 3. Flags that user might want to review - // ---------------------------------------- - // The following affect all levels of LSM tree. - MaxTableSize int64 // Each table (or file) is at most this size. - LevelSizeMultiplier int // Equals SizeOf(Li+1)/SizeOf(Li). - MaxLevels int // Maximum number of levels of compaction. - // If value size >= this threshold, only store value offsets in tree. - ValueThreshold int - // Maximum number of tables to keep in memory, before stalling. - NumMemtables int - // The following affect how we handle LSM tree L0. - // Maximum number of Level 0 tables before we start compacting. - NumLevelZeroTables int - - // If we hit this number of Level 0 tables, we will stall until L0 is - // compacted away. - NumLevelZeroTablesStall int + // Fine tuning options. - // Maximum total size for L1. - LevelOneSize int64 + MaxTableSize int64 + LevelSizeMultiplier int + MaxLevels int + ValueThreshold int + NumMemtables int - // Size of single value log file. - ValueLogFileSize int64 + NumLevelZeroTables int + NumLevelZeroTablesStall int - // Max number of entries a value log file can hold (approximately). A value log file would be - // determined by the smaller of its file size and max entries. + LevelOneSize int64 + ValueLogFileSize int64 ValueLogMaxEntries uint32 - // Number of compaction workers to run concurrently. Setting this to zero would stop compactions - // to happen within LSM tree. If set to zero, writes could block forever. - NumCompactors int - - // When closing the DB, force compact Level 0. This ensures that both reads and writes are - // efficient when the DB is opened later. - CompactL0OnClose bool - - // After this many number of value log file rotates, there would be a force flushing of memtable - // to disk. This is useful in write loads with fewer keys and larger values. This work load - // would fill up the value logs quickly, while not filling up the Memtables. Thus, on a crash - // and restart, the value log head could cause the replay of a good number of value log files - // which can slow things on start. + NumCompactors int + CompactL0OnClose bool LogRotatesToFlush int32 // Transaction start and commit timestamps are managed by end-user. @@ -121,46 +77,46 @@ type Options struct { } // DefaultOptions sets a list of recommended options for good performance. -// Feel free to modify these to suit your needs. -var DefaultOptions = Options{ - LevelOneSize: 256 << 20, - LevelSizeMultiplier: 10, - TableLoadingMode: options.MemoryMap, - ValueLogLoadingMode: options.MemoryMap, - // table.MemoryMap to mmap() the tables. - // table.Nothing to not preload the tables. - MaxLevels: 7, - MaxTableSize: 64 << 20, - NumCompactors: 2, // Compactions can be expensive. Only run 2. - NumLevelZeroTables: 5, - NumLevelZeroTablesStall: 10, - NumMemtables: 5, - SyncWrites: true, - NumVersionsToKeep: 1, - CompactL0OnClose: true, - // Nothing to read/write value log using standard File I/O - // MemoryMap to mmap() the value log files - // (2^30 - 1)*2 when mmapping < 2^31 - 1, max int32. - // -1 so 2*ValueLogFileSize won't overflow on 32-bit systems. - ValueLogFileSize: 1<<30 - 1, - - ValueLogMaxEntries: 1000000, - ValueThreshold: 32, - Truncate: false, - Logger: defaultLogger, - LogRotatesToFlush: 2, +// Feel free to modify these to suit your needs with the WithX methods. +func DefaultOptions(path string) Options { + return Options{ + Dir: path, + ValueDir: path, + LevelOneSize: 256 << 20, + LevelSizeMultiplier: 10, + TableLoadingMode: options.MemoryMap, + ValueLogLoadingMode: options.MemoryMap, + // table.MemoryMap to mmap() the tables. + // table.Nothing to not preload the tables. + MaxLevels: 7, + MaxTableSize: 64 << 20, + NumCompactors: 2, // Compactions can be expensive. Only run 2. + NumLevelZeroTables: 5, + NumLevelZeroTablesStall: 10, + NumMemtables: 5, + SyncWrites: true, + NumVersionsToKeep: 1, + CompactL0OnClose: true, + // Nothing to read/write value log using standard File I/O + // MemoryMap to mmap() the value log files + // (2^30 - 1)*2 when mmapping < 2^31 - 1, max int32. + // -1 so 2*ValueLogFileSize won't overflow on 32-bit systems. + ValueLogFileSize: 1<<30 - 1, + + ValueLogMaxEntries: 1000000, + ValueThreshold: 32, + Truncate: false, + Logger: defaultLogger, + LogRotatesToFlush: 2, + } } // LSMOnlyOptions follows from DefaultOptions, but sets a higher ValueThreshold // so values would be colocated with the LSM tree, with value log largely acting // as a write-ahead log only. These options would reduce the disk usage of value // log, and make Badger act more like a typical LSM tree. -var LSMOnlyOptions = Options{} - -func init() { - LSMOnlyOptions = DefaultOptions - - LSMOnlyOptions.ValueThreshold = 65500 // Max value length which fits in uint16. +func LSMOnlyOptions(path string) Options { + // Max value length which fits in uint16. // Let's not set any other options, because they can cause issues with the // size of key-value a user can pass to Badger. For e.g., if we set // ValueLogFileSize to 64MB, a user can't pass a value more than that. @@ -171,4 +127,248 @@ func init() { // NOTE: If a user does not want to set 64KB as the ValueThreshold because // of performance reasons, 1KB would be a good option too, allowing // values smaller than 1KB to be colocated with the keys in the LSM tree. + return DefaultOptions(path).WithValueThreshold(65500) +} + +// WithDir returns a new Options value with Dir set to the given value. +// +// Dir is the path of the directory where key data will be stored in. +// If it doesn't exist, Badger will try to create it for you. +// This is set automatically to be the path given to `DefaultOptions`. +func (opt Options) WithDir(val string) Options { + opt.Dir = val + return opt +} + +// WithValueDir returns a new Options value with ValueDir set to the given value. +// +// ValueDir is the path of the directory where value data will be stored in. +// If it doesn't exist, Badger will try to create it for you. +// This is set automatically to be the path given to `DefaultOptions`. +func (opt Options) WithValueDir(val string) Options { + opt.ValueDir = val + return opt +} + +// WithSyncWrites returns a new Options value with SyncWrites set to the given value. +// +// When SyncWrites is true all writes are synced to disk. Setting this to false would achieve better +// performance, but may cause data loss in case of crash. +// +// The default value of SyncWrites is true. +func (opt Options) WithSyncWrites(val bool) Options { + opt.SyncWrites = val + return opt +} + +// WithTableLoadingMode returns a new Options value with TableLoadingMode set to the given value. +// +// TableLoadingMode indicates which file loading mode should be used for the LSM tree data files. +// +// The default value of TableLoadingMode is options.MemoryMap. +func (opt Options) WithTableLoadingMode(val options.FileLoadingMode) Options { + opt.TableLoadingMode = val + return opt +} + +// WithValueLogLoadingMode returns a new Options value with ValueLogLoadingMode set to the given +// value. +// +// ValueLogLoadingMode indicates which file loading mode should be used for the value log data +// files. +// +// The default value of ValueLogLoadingMode is options.MemoryMap. +func (opt Options) WithValueLogLoadingMode(val options.FileLoadingMode) Options { + opt.ValueLogLoadingMode = val + return opt +} + +// WithNumVersionsToKeep returns a new Options value with NumVersionsToKeep set to the given value. +// +// NumVersionsToKeep sets how many versions to keep per key at most. +// +// The default value of NumVersionsToKeep is 1. +func (opt Options) WithNumVersionsToKeep(val int) Options { + opt.NumVersionsToKeep = val + return opt +} + +// WithReadOnly returns a new Options value with ReadOnly set to the given value. +// +// When ReadOnly is true the DB will be opened on read-only mode. +// Multiple processes can open the same Badger DB. +// Note: if the DB being opened had crashed before and has vlog data to be replayed, +// ReadOnly will cause Open to fail with an appropriate message. +// +// The default value of ReadOnly is false. +func (opt Options) WithReadOnly(val bool) Options { + opt.ReadOnly = val + return opt +} + +// WithTruncate returns a new Options value with Truncate set to the given value. +// +// Truncate indicates whether value log files should be truncated to delete corrupt data, if any. +// This option is ignored when ReadOnly is true. +// +// The default value of Truncate is false. +func (opt Options) WithTruncate(val bool) Options { + opt.Truncate = val + return opt +} + +// WithLogger returns a new Options value with Logger set to the given value. +// +// Logger provides a way to configure what logger each value of badger.DB uses. +// +// The default value of Logger writes to stderr using the log package from the Go standard library. +func (opt Options) WithLogger(val Logger) Options { + opt.Logger = val + return opt +} + +// WithMaxTableSize returns a new Options value with MaxTableSize set to the given value. +// +// MaxTableSize sets the maximum size in bytes for each LSM table or file. +// +// The default value of MaxTableSize is 64MB. +func (opt Options) WithMaxTableSize(val int64) Options { + opt.MaxTableSize = val + return opt +} + +// WithLevelSizeMultiplier returns a new Options value with LevelSizeMultiplier set to the given +// value. +// +// LevelSizeMultiplier sets the ratio between the maximum sizes of contiguous levels in the LSM. +// Once a level grows to be larger than this ratio allowed, the compaction process will be +// triggered. +// +// The default value of LevelSizeMultiplier is 10. +func (opt Options) WithLevelSizeMultiplier(val int) Options { + opt.LevelSizeMultiplier = val + return opt +} + +// WithMaxLevels returns a new Options value with MaxLevels set to the given value. +// +// Maximum number of levels of compaction allowed in the LSM. +// +// The default value of MaxLevels is 7. +func (opt Options) WithMaxLevels(val int) Options { + opt.MaxLevels = val + return opt +} + +// WithValueThreshold returns a new Options value with ValueThreshold set to the given value. +// +// ValueThreshold sets the threshold used to decide whether a value is stored directly in the LSM +// tree or separatedly in the log value files. +// +// The default value of ValueThreshold is 32, but LSMOnlyOptions sets it to 65500. +func (opt Options) WithValueThreshold(val int) Options { + opt.ValueThreshold = val + return opt +} + +// WithNumMemtables returns a new Options value with NumMemtables set to the given value. +// +// NumMemtables sets the maximum number of tables to keep in memory before stalling. +// +// The default value of NumMemtables is 5. +func (opt Options) WithNumMemtables(val int) Options { + opt.NumMemtables = val + return opt +} + +// WithNumLevelZeroTables returns a new Options value with NumLevelZeroTables set to the given +// value. +// +// NumLevelZeroTables sets the maximum number of Level 0 tables before compaction starts. +// +// The default value of NumLevelZeroTables is 5. +func (opt Options) WithNumLevelZeroTables(val int) Options { + opt.NumLevelZeroTables = val + return opt +} + +// WithNumLevelZeroTablesStall returns a new Options value with NumLevelZeroTablesStall set to the +// given value. +// +// NumLevelZeroTablesStall sets the number of Level 0 tables that once reached causes the DB to +// stall until compaction succeeds. +// +// The default value of NumLevelZeroTablesStall is 10. +func (opt Options) WithNumLevelZeroTablesStall(val int) Options { + opt.NumLevelZeroTablesStall = val + return opt +} + +// WithLevelOneSize returns a new Options value with LevelOneSize set to the given value. +// +// LevelOneSize sets the maximum total size for Level 1. +// +// The default value of LevelOneSize is 20MB. +func (opt Options) WithLevelOneSize(val int64) Options { + opt.LevelOneSize = val + return opt +} + +// WithValueLogFileSize returns a new Options value with ValueLogFileSize set to the given value. +// +// ValueLogFileSize sets the maximum size of a single value log file. +// +// The default value of ValueLogFileSize is 1GB. +func (opt Options) WithValueLogFileSize(val int64) Options { + opt.ValueLogFileSize = val + return opt +} + +// WithValueLogMaxEntries returns a new Options value with ValueLogMaxEntries set to the given +// value. +// +// ValueLogMaxEntries sets the maximum number of entries a value log file can hold approximately. +// A actual size limit of a value log file is the minimum of ValueLogFileSize and +// ValueLogMaxEntries. +// +// The default value of ValueLogMaxEntries is one million (1000000). +func (opt Options) WithValueLogMaxEntries(val uint32) Options { + opt.ValueLogMaxEntries = val + return opt +} + +// WithNumCompactors returns a new Options value with NumCompactors set to the given value. +// +// NumCompactors sets the number of compaction workers to run concurrently. +// Setting this to zero stops compactions, which could eventually cause writes to block forever. +// +// The default value of NumCompactors is 2. +func (opt Options) WithNumCompactors(val int) Options { + opt.NumCompactors = val + return opt +} + +// WithCompactL0OnClose returns a new Options value with CompactL0OnClose set to the given value. +// +// CompactL0OnClose determines whether Level 0 should be compacted before closing the DB. +// This ensures that both reads and writes are efficient when the DB is opened later. +// +// The default value of CompactL0OnClose is true. +func (opt Options) WithCompactL0OnClose(val bool) Options { + opt.CompactL0OnClose = val + return opt +} + +// WithLogRotatesToFlush returns a new Options value with LogRotatesToFlush set to the given value. +// +// LogRotatesToFlush sets the number of value log file rotates after which the Memtables are +// flushed to disk. This is useful in write loads with fewer keys and larger values. This work load +// would fill up the value logs quickly, while not filling up the Memtables. Thus, on a crash +// and restart, the value log head could cause the replay of a good number of value log files +// which can slow things on start. +// +// The default value of LogRotatesToFlush is 2. +func (opt Options) WithLogRotatesToFlush(val int32) Options { + opt.LogRotatesToFlush = val + return opt } diff --git a/stream_test.go b/stream_test.go index 85943a279..33c00df39 100644 --- a/stream_test.go +++ b/stream_test.go @@ -31,14 +31,6 @@ import ( "github.com/stretchr/testify/require" ) -func openManaged(dir string) (*DB, error) { - opt := DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - - return OpenManaged(opt) -} - func keyWithPrefix(prefix string, k int) []byte { return []byte(fmt.Sprintf("%s-%d", prefix, k)) } @@ -70,7 +62,7 @@ func TestStream(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - db, err := openManaged(dir) + db, err := OpenManaged(DefaultOptions(dir)) require.NoError(t, err) var count int diff --git a/stream_writer_test.go b/stream_writer_test.go index f77eddca1..cec2df50f 100644 --- a/stream_writer_test.go +++ b/stream_writer_test.go @@ -49,8 +49,8 @@ func getSortedKVList(valueSize, listSize int) *pb.KVList { // check if we can read values after writing using stream writer func TestStreamWriter1(t *testing.T) { - normalModeOpts := DefaultOptions - managedModeOpts := DefaultOptions + normalModeOpts := DefaultOptions("") + managedModeOpts := DefaultOptions("") managedModeOpts.managedTxns = true for _, opts := range []*Options{&normalModeOpts, &managedModeOpts} { @@ -90,8 +90,8 @@ func TestStreamWriter1(t *testing.T) { // write more keys to db after writing keys using stream writer func TestStreamWriter2(t *testing.T) { - normalModeOpts := DefaultOptions - managedModeOpts := DefaultOptions + normalModeOpts := DefaultOptions("") + managedModeOpts := DefaultOptions("") managedModeOpts.managedTxns = true for _, opts := range []*Options{&normalModeOpts, &managedModeOpts} { @@ -142,8 +142,8 @@ func TestStreamWriter2(t *testing.T) { } func TestStreamWriter3(t *testing.T) { - normalModeOpts := DefaultOptions - managedModeOpts := DefaultOptions + normalModeOpts := DefaultOptions("") + managedModeOpts := DefaultOptions("") managedModeOpts.managedTxns = true for _, opts := range []*Options{&normalModeOpts, &managedModeOpts} { diff --git a/txn_test.go b/txn_test.go index b851b5776..7a3c12543 100644 --- a/txn_test.go +++ b/txn_test.go @@ -803,16 +803,12 @@ func TestArmV7Issue311Fix(t *testing.T) { } defer os.RemoveAll(dir) - config := DefaultOptions - config.TableLoadingMode = options.MemoryMap - config.ValueLogFileSize = 16 << 20 - config.LevelOneSize = 8 << 20 - config.MaxTableSize = 2 << 20 - config.Dir = dir - config.ValueDir = dir - config.SyncWrites = false - - db, err := Open(config) + db, err := Open(DefaultOptions(dir). + WithTableLoadingMode(options.MemoryMap). + WithValueLogFileSize(16 << 20). + WithLevelOneSize(8 << 20). + WithMaxTableSize(2 << 20). + WithSyncWrites(false)) if err != nil { t.Fatalf("cannot open db at location %s: %v", dir, err) } diff --git a/value_test.go b/value_test.go index 4b9c7548a..08a96f6e9 100644 --- a/value_test.go +++ b/value_test.go @@ -844,13 +844,9 @@ func TestBug578(t *testing.T) { y.Check(err) defer os.RemoveAll(dir) - opts := DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - opts.ValueLogMaxEntries = 64 - opts.MaxTableSize = 1 << 13 - - db, err := Open(opts) + db, err := Open(DefaultOptions(dir). + WithValueLogMaxEntries(64). + WithMaxTableSize(1 << 13)) require.NoError(t, err) h := testHelper{db: db, t: t} @@ -942,12 +938,7 @@ func TestValueLogTruncate(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - opts := DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - opts.Truncate = true - - db, err := Open(opts) + db, err := Open(DefaultOptions(dir).WithTruncate(true)) require.NoError(t, err) // Insert 1 entry so that we have valid data in first vlog file require.NoError(t, db.Update(func(txn *Txn) error { @@ -962,7 +953,7 @@ func TestValueLogTruncate(t *testing.T) { require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 1), []byte("foo"), 0664)) require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 2), []byte("foo"), 0664)) - db, err = Open(opts) + db, err = Open(DefaultOptions(dir).WithTruncate(true)) require.NoError(t, err) // Ensure vlog file with id=1 is not present