Store total key-value size in table footer

levels.go

@@ -1014,6 +1014,7 @@ type TableInfo struct {
 	Left     []byte
 	Right    []byte
 	KeyCount uint64 // Number of keys in the table
+	KVSize   uint64
 }
 
 func (s *levelsController) getTableInfo(withKeysCount bool) (result []TableInfo) {
@@ -1035,6 +1036,7 @@ func (s *levelsController) getTableInfo(withKeysCount bool) (result []TableInfo)
 				Left:     t.Smallest(),
 				Right:    t.Biggest(),
 				KeyCount: count,
+				KVSize:   t.TotalKVSize(),
 			}
 			result = append(result, info)
 		}

pb/pb.proto

@@ -43,7 +43,7 @@ message ManifestChangeSet {
 }
 
 enum EncryptionAlgo {
-                aes = 0;  
+  aes = 0;
 }
 
 message ManifestChange {
@@ -68,6 +68,7 @@ message BlockOffset {
 message TableIndex {
   repeated BlockOffset offsets = 1;
   bytes bloom_filter = 2;
+  uint64 kv_size = 3;
 }
 
 message Checksum {
@@ -84,4 +85,4 @@ message DataKey {
   bytes  data       = 2;
   bytes  iv         = 3;
   int64  created_at = 4;
-}
+}

table/builder.go

@@ -69,11 +69,9 @@ type Builder struct {
 	baseKey      []byte   // Base key for the current block.
 	baseOffset   uint32   // Offset for the current block.
 	entryOffsets []uint32 // Offsets of entries present in current block.
-
-	tableIndex *pb.TableIndex
-	keyHashes  []uint64
-
-	opt *Options
+	tableIndex   *pb.TableIndex
+	keyHashes    []uint64 // Used for building bloomfilter.
+	opt          *Options
 }
 
 // NewTableBuilder makes a new TableBuilder.
@@ -131,6 +129,7 @@ func (b *Builder) addHelper(key []byte, v y.ValueStruct) {
 	b.buf.Write(diffKey) // We only need to store the key difference.
 
 	v.EncodeTo(b.buf)
+	b.tableIndex.KvSize += uint64(uint32(headerSize) + uint32(len(diffKey)) + v.EncodedSize())
 }
 
 /*

table/builder_test.go

@@ -135,3 +135,16 @@ func BenchmarkBuilder(b *testing.B) {
 		_ = builder.Finish()
 	}
 }
+
+func TestKVSize(t *testing.T) {
+	opts := Options{BlockSize: 4 * 1024, BloomFalsePositive: 0.01}
+	b := NewTableBuilder(opts)
+	defer b.Close()
+
+	k := y.KeyWithTs([]byte(key("foo", 0)), 0)
+	v := []byte(fmt.Sprintf("%d", 0))
+
+	b.Add(k, y.ValueStruct{Value: v, Meta: 'A', UserMeta: 0})
+	var entrySize uint64 = 15 /* DiffKey len */ + 4 /* Header Size */ + 4 /* Encoded vp */
+	require.Equal(t, b.tableIndex.KvSize, entrySize)
+}

table/table.go

@@ -99,6 +99,7 @@ type Table struct {
 
 	bf       *z.Bloom
 	Checksum []byte
+	kvSize   uint64 // Stores the total size of key-values stored in this table.
 
 	IsInmemory bool // Set to true if the table is on level 0 and opened in memory.
 	opt        *Options
@@ -351,6 +352,7 @@ func (t *Table) readIndex() error {
 	err := proto.Unmarshal(data, &index)
 	y.Check(err)
 
+	t.kvSize = index.KvSize
 	t.bf = z.JSONUnmarshal(index.BloomFilter)
 	t.blockIndex = index.Offsets
 	return nil
@@ -439,6 +441,9 @@ func (t *Table) blockCacheKey(idx int) uint64 {
 	return (t.ID() << 32) | uint64(idx)
 }
 
+// TotalKVSize returns the total size of key-values stored in this table.
+func (t *Table) TotalKVSize() uint64 { return t.kvSize }
+
 // Size is its file size in bytes
 func (t *Table) Size() int64 { return int64(t.tableSize) }
 

table/table_test.go

@@ -927,3 +927,14 @@ func TestMain(m *testing.M) {
 	rand.Seed(time.Now().UTC().UnixNano())
 	os.Exit(m.Run())
 }
+
+func TestOpenKVSize(t *testing.T) {
+	opts := getTestTableOptions()
+	f := buildTestTable(t, "foo", 1, opts)
+
+	table, err := OpenTable(f, opts)
+	require.NoError(t, err)
+
+	var entrySize uint64 = 15 /* DiffKey len */ + 4 /* Header Size */ + 4 /* Encoded vp */
+	require.Equal(t, entrySize, table.kvSize)
+}