feat(vlog): Remove Value Log (#11)

We haven't used the value log for quite some time in Dgraph. And Outserv doesn't use value log either. As part of bringing Badger back to the core features and shedding its weight, I'm removing the value log.

badger/backup.go

@@ -192,7 +192,7 @@ func (l *KVLoader) Set(kv *pb.KV) error {
  ExpiresAt: kv.ExpiresAt,
  meta: meta,
  }
- estimatedSize := e.estimateSizeAndSetThreshold(l.db.valueThreshold())
+ estimatedSize := e.estimateSize()
  // Flush entries if inserting the next entry would overflow the transactional limits.
  if int64(len(l.entries))+1 >= l.db.opt.maxBatchCount ||
  l.entriesSize+estimatedSize >= l.db.opt.maxBatchSize ||

badger/backup_test.go

@@ -512,13 +512,11 @@ func TestBackupBitClear(t *testing.T) {
  defer removeDir(dir)
 
  opt := getTestOptions(dir)
- opt.ValueThreshold = 10 // This is important
  db, err := Open(opt)
  require.NoError(t, err)
 
  key := []byte("foo")
  val := []byte(fmt.Sprintf("%0100d", 1))
- require.Greater(t, int64(len(val)), db.valueThreshold())
 
  err = db.Update(func(txn *Txn) error {
  e := NewEntry(key, val)
@@ -542,7 +540,6 @@ func TestBackupBitClear(t *testing.T) {
  require.NoError(t, db.Close())
 
  opt = getTestOptions(dir)
- opt.ValueThreshold = 200 // This is important.
  db, err = Open(opt)
  require.NoError(t, err)
  defer db.Close()

badger/badger/cmd/info.go

@@ -50,7 +50,6 @@ type flagOptions struct {
  truncate bool
  encryptionKey string
  checksumVerificationMode string
- discard bool
  externalMagicVersion uint16
 }
 
@@ -81,8 +80,6 @@ func init() {
  infoCmd.Flags().StringVar(&opt.encryptionKey, "enc-key", "", "Use the provided encryption key")
  infoCmd.Flags().StringVar(&opt.checksumVerificationMode, "cv-mode", "none",
  "[none, table, block, tableAndBlock] Specifies when the db should verify checksum for SST.")
- infoCmd.Flags().BoolVar(&opt.discard, "discard", false,
- "Parse and print DISCARD file from value logs.")
  infoCmd.Flags().Uint16Var(&opt.externalMagicVersion, "external-magic", 0,
  "External magic number")
 }
@@ -110,17 +107,6 @@ func handleInfo(cmd *cobra.Command, args []string) error {
  WithChecksumVerificationMode(cvMode).
  WithExternalMagic(opt.externalMagicVersion)
 
- if opt.discard {
- ds, err := badger.InitDiscardStats(bopt)
- y.Check(err)
- ds.Iterate(func(fid, stats uint64) {
- fmt.Printf("Value Log Fid: %5d. Stats: %10d [ %s ]\n",
- fid, stats, humanize.IBytes(stats))
- })
- fmt.Println("DONE")
- return nil
- }
-
  if err := printInfo(sstDir, vlogDir); err != nil {
  return y.Wrap(err, "failed to print information in MANIFEST file")
  }

badger/badger/cmd/pick_table_bench.go

@@ -192,8 +192,8 @@ func genTables(boundaries [][]byte) []*table.Table {
  b := table.NewTableBuilder(opts)
  defer b.Close()
  // Add one key so that we can open this table.
- b.Add(y.KeyWithTs(k1, 1), y.ValueStruct{}, 0)
- b.Add(y.KeyWithTs(k2, 1), y.ValueStruct{}, 0)
+ b.Add(y.KeyWithTs(k1, 1), y.ValueStruct{})
+ b.Add(y.KeyWithTs(k2, 1), y.ValueStruct{})
  tab, err := table.OpenInMemoryTable(b.Finish(), 0, &opts)
  y.Check(err)
  return tab

badger/badger/cmd/stream.go

@@ -77,7 +77,6 @@ func stream(cmd *cobra.Command, args []string) error {
  }
  inOpt := badger.DefaultOptions(sstDir).
  WithReadOnly(so.readOnly).
- WithValueThreshold(1 << 10 /* 1KB */).
  WithNumVersionsToKeep(so.numVersions).
  WithBlockCacheSize(100 << 20).
  WithIndexCacheSize(200 << 20).

badger/badger/cmd/write_bench.go

@@ -269,11 +269,9 @@ func writeBench(cmd *cobra.Command, args []string) error {
  WithValueDir(vlogDir).
  WithSyncWrites(wo.syncWrites).
  WithCompactL0OnClose(wo.force).
- WithValueThreshold(wo.valueThreshold).
  WithNumVersionsToKeep(wo.numVersions).
  WithBlockCacheSize(wo.blockCacheSize << 20).
  WithIndexCacheSize(wo.indexCacheSize << 20).
- WithValueLogMaxEntries(wo.vlogMaxEntries).
  WithEncryptionKey([]byte(wo.encryptionKey)).
  WithDetectConflicts(wo.detectConflicts).
  WithLoggingLevel(badger.INFO)
@@ -302,11 +300,10 @@ func writeBench(cmd *cobra.Command, args []string) error {
 
  startTime = time.Now()
  num := uint64(wo.numKeys * mil)
- c := z.NewCloser(4)
+ c := z.NewCloser(3)
  go reportStats(c, db)
  go dropAll(c, db)
  go dropPrefix(c, db)
- go runGC(c, db)
 
  if wo.sorted {
  err = writeSorted(db, num)
@@ -408,30 +405,6 @@ func reportStats(c *z.Closer, db *badger.DB) {
  }
 }
 
-func runGC(c *z.Closer, db *badger.DB) {
- defer c.Done()
- period, err := time.ParseDuration(wo.gcPeriod)
- y.Check(err)
- if period == 0 {
- return
- }
-
- t := time.NewTicker(period)
- defer t.Stop()
- for {
- select {
- case <-c.HasBeenClosed():
- return
- case <-t.C:
- if err := db.RunValueLogGC(wo.gcDiscardRatio); err == nil {
- atomic.AddUint64(&gcSuccess, 1)
- } else {
- log.Printf("[GC] Failed due to following err %v", err)
- }
- }
- }
-}
-
 func dropAll(c *z.Closer, db *badger.DB) {
  defer c.Done()
  dropPeriod, err := time.ParseDuration(wo.dropAllPeriod)

badger/batch_test.go

@@ -74,9 +74,6 @@ func TestWriteBatch(t *testing.T) {
  }
  t.Run("disk mode", func(t *testing.T) {
  opt := getTestOptions("")
- // Set value threshold to 32 bytes otherwise write batch will generate
- // too many files and we will crash with too many files open error.
- opt.ValueThreshold = 32
  runBadgerTest(t, &opt, func(t *testing.T, db *DB) {
  test(t, db)
  })