Skip to content

Commit e77db04

Browse files
roseduanroseduan
committed
remove block cache
1 parent 8de9190 commit e77db04

File tree

6 files changed

+22
-96
lines changed

6 files changed

+22
-96
lines changed

‎go.mod

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ require (
99

1010
require (
1111
github.com/davecgh/go-spew v1.1.1 // indirect
12-
github.com/hashicorp/golang-lru/v2 v2.0.2
1312
github.com/pmezard/go-difflib v1.0.0 // indirect
1413
gopkg.in/yaml.v3 v3.0.1 // indirect
1514
)

‎options.go

-6
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,6 @@ type Options struct {
1717
// Not a common usage for most users.
1818
SegmentFileExt string
1919

20-
// BlockCache specifies the size of the block cache in number of bytes.
21-
// A block cache is used to store recently accessed data blocks, improving read performance.
22-
// If BlockCache is set to 0, no block cache will be used.
23-
BlockCache uint32
24-
2520
// Sync is whether to synchronize writes through os buffer cache and down onto the actual disk.
2621
// Setting sync is required for durability of a single write operation, but also results in slower writes.
2722
//
@@ -47,7 +42,6 @@ var DefaultOptions = Options{
4742
DirPath: os.TempDir(),
4843
SegmentSize: GB,
4944
SegmentFileExt: ".SEG",
50-
BlockCache: 32 * KB * 10,
5145
Sync: false,
5246
BytesPerSync: 0,
5347
}

‎segment.go

+5-31
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"os"
1010
"sync"
1111

12-
lru "github.com/hashicorp/golang-lru/v2"
1312
"github.com/valyala/bytebufferpool"
1413
)
1514

@@ -53,7 +52,6 @@ type segment struct {
5352
currentBlockNumber uint32
5453
currentBlockSize uint32
5554
closed bool
56-
cache *lru.Cache[uint64, []byte]
5755
header []byte
5856
blockPool sync.Pool
5957
}
@@ -86,7 +84,7 @@ type ChunkPosition struct {
8684
}
8785

8886
// openSegmentFile a new segment file.
89-
func openSegmentFile(dirPath, extName string, id uint32, cache *lru.Cache[uint64, []byte]) (*segment, error) {
87+
func openSegmentFile(dirPath, extName string, id uint32) (*segment, error) {
9088
fd, err := os.OpenFile(
9189
SegmentFileName(dirPath, extName, id),
9290
os.O_CREATE|os.O_RDWR|os.O_APPEND,
@@ -106,7 +104,6 @@ func openSegmentFile(dirPath, extName string, id uint32, cache *lru.Cache[uint64
106104
return &segment{
107105
id: id,
108106
fd: fd,
109-
cache: cache,
110107
header: make([]byte, chunkHeaderSize),
111108
blockPool: sync.Pool{New: newBlockAndHeader},
112109
currentBlockNumber: uint32(offset / blockSize),
@@ -394,29 +391,10 @@ func (seg *segment) readInternal(blockNumber uint32, chunkOffset int64) ([]byte,
394391
return nil, nil, io.EOF
395392
}
396393

397-
var ok bool
398-
var cachedBlock []byte
399-
// try to read from the cache if it is enabled
400-
if seg.cache != nil {
401-
cachedBlock, ok = seg.cache.Get(seg.getCacheKey(blockNumber))
402-
}
403-
// cache hit, get block from the cache
404-
if ok {
405-
copy(bh.block, cachedBlock)
406-
} else {
407-
// cache miss, read block from the segment file
408-
_, err := seg.fd.ReadAt(bh.block[0:size], offset)
409-
if err != nil {
410-
return nil, nil, err
411-
}
412-
// cache the block, so that the next time it can be read from the cache.
413-
// if the block size is smaller than blockSize, it means that the block is not full,
414-
// so we will not cache it.
415-
if seg.cache != nil && size == blockSize && len(cachedBlock) == 0 {
416-
cacheBlock := make([]byte, blockSize)
417-
copy(cacheBlock, bh.block)
418-
seg.cache.Add(seg.getCacheKey(blockNumber), cacheBlock)
419-
}
394+
// cache miss, read block from the segment file
395+
_, err := seg.fd.ReadAt(bh.block[0:size], offset)
396+
if err != nil {
397+
return nil, nil, err
420398
}
421399

422400
// header
@@ -457,10 +435,6 @@ func (seg *segment) readInternal(blockNumber uint32, chunkOffset int64) ([]byte,
457435
return result, nextChunk, nil
458436
}
459437

460-
func (seg *segment) getCacheKey(blockNumber uint32) uint64 {
461-
return uint64(seg.id)<<32 | uint64(blockNumber)
462-
}
463-
464438
// Next returns the Next chunk data.
465439
// You can call it repeatedly until io.EOF is returned.
466440
func (segReader *segmentReader) Next() ([]byte, *ChunkPosition, error) {

‎segment_test.go

+12-16
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,12 @@ import (
88
"strings"
99
"testing"
1010

11-
lru "github.com/hashicorp/golang-lru/v2"
1211
"github.com/stretchr/testify/assert"
1312
)
1413

1514
func TestSegment_Write_FULL1(t *testing.T) {
1615
dir, _ := os.MkdirTemp("", "seg-test-full1")
17-
seg, err := openSegmentFile(dir, ".SEG", 1, nil)
16+
seg, err := openSegmentFile(dir, ".SEG", 1)
1817
assert.Nil(t, err)
1918
defer func() {
2019
_ = seg.Remove()
@@ -40,15 +39,15 @@ func TestSegment_Write_FULL1(t *testing.T) {
4039
for i := 0; i < 100000; i++ {
4140
pos, err := seg.Write(val)
4241
assert.Nil(t, err)
43-
val, err := seg.Read(pos.BlockNumber, pos.ChunkOffset)
42+
res, err := seg.Read(pos.BlockNumber, pos.ChunkOffset)
4443
assert.Nil(t, err)
45-
assert.Equal(t, val, val)
44+
assert.Equal(t, val, res)
4645
}
4746
}
4847

4948
func TestSegment_Write_FULL2(t *testing.T) {
5049
dir, _ := os.MkdirTemp("", "seg-test-full2")
51-
seg, err := openSegmentFile(dir, ".SEG", 1, nil)
50+
seg, err := openSegmentFile(dir, ".SEG", 1)
5251
assert.Nil(t, err)
5352
defer func() {
5453
_ = seg.Remove()
@@ -76,7 +75,7 @@ func TestSegment_Write_FULL2(t *testing.T) {
7675

7776
func TestSegment_Write_Padding(t *testing.T) {
7877
dir, _ := os.MkdirTemp("", "seg-test-padding")
79-
seg, err := openSegmentFile(dir, ".SEG", 1, nil)
78+
seg, err := openSegmentFile(dir, ".SEG", 1)
8079
assert.Nil(t, err)
8180
defer func() {
8281
_ = seg.Remove()
@@ -99,7 +98,7 @@ func TestSegment_Write_Padding(t *testing.T) {
9998

10099
func TestSegment_Write_NOT_FULL(t *testing.T) {
101100
dir, _ := os.MkdirTemp("", "seg-test-not-full")
102-
seg, err := openSegmentFile(dir, ".SEG", 1, nil)
101+
seg, err := openSegmentFile(dir, ".SEG", 1)
103102
assert.Nil(t, err)
104103
defer func() {
105104
_ = seg.Remove()
@@ -137,7 +136,7 @@ func TestSegment_Write_NOT_FULL(t *testing.T) {
137136

138137
func TestSegment_Reader_FULL(t *testing.T) {
139138
dir, _ := os.MkdirTemp("", "seg-test-reader-full")
140-
seg, err := openSegmentFile(dir, ".SEG", 1, nil)
139+
seg, err := openSegmentFile(dir, ".SEG", 1)
141140
assert.Nil(t, err)
142141
defer func() {
143142
_ = seg.Remove()
@@ -169,7 +168,7 @@ func TestSegment_Reader_FULL(t *testing.T) {
169168

170169
func TestSegment_Reader_Padding(t *testing.T) {
171170
dir, _ := os.MkdirTemp("", "seg-test-reader-padding")
172-
seg, err := openSegmentFile(dir, ".SEG", 1, nil)
171+
seg, err := openSegmentFile(dir, ".SEG", 1)
173172
assert.Nil(t, err)
174173
defer func() {
175174
_ = seg.Remove()
@@ -203,7 +202,7 @@ func TestSegment_Reader_Padding(t *testing.T) {
203202

204203
func TestSegment_Reader_NOT_FULL(t *testing.T) {
205204
dir, _ := os.MkdirTemp("", "seg-test-reader-not-full")
206-
seg, err := openSegmentFile(dir, ".SEG", 1, nil)
205+
seg, err := openSegmentFile(dir, ".SEG", 1)
207206
assert.Nil(t, err)
208207
defer func() {
209208
_ = seg.Remove()
@@ -249,8 +248,7 @@ func TestSegment_Reader_NOT_FULL(t *testing.T) {
249248

250249
func TestSegment_Reader_ManyChunks_FULL(t *testing.T) {
251250
dir, _ := os.MkdirTemp("", "seg-test-reader-ManyChunks_FULL")
252-
cache, _ := lru.New[uint64, []byte](5)
253-
seg, err := openSegmentFile(dir, ".SEG", 1, cache)
251+
seg, err := openSegmentFile(dir, ".SEG", 1)
254252
assert.Nil(t, err)
255253
defer func() {
256254
_ = seg.Remove()
@@ -287,8 +285,7 @@ func TestSegment_Reader_ManyChunks_FULL(t *testing.T) {
287285

288286
func TestSegment_Reader_ManyChunks_NOT_FULL(t *testing.T) {
289287
dir, _ := os.MkdirTemp("", "seg-test-reader-ManyChunks_NOT_FULL")
290-
cache, _ := lru.New[uint64, []byte](5)
291-
seg, err := openSegmentFile(dir, ".SEG", 1, cache)
288+
seg, err := openSegmentFile(dir, ".SEG", 1)
292289
assert.Nil(t, err)
293290
defer func() {
294291
_ = seg.Remove()
@@ -337,8 +334,7 @@ func TestSegment_Write_LargeSize(t *testing.T) {
337334

338335
func testSegmentReaderLargeSize(t *testing.T, size int, count int) {
339336
dir, _ := os.MkdirTemp("", "seg-test-reader-ManyChunks_large_size")
340-
cache, _ := lru.New[uint64, []byte](5)
341-
seg, err := openSegmentFile(dir, ".SEG", 1, cache)
337+
seg, err := openSegmentFile(dir, ".SEG", 1)
342338
assert.Nil(t, err)
343339
defer func() {
344340
_ = seg.Remove()

‎wal.go

+4-33
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ import (
99
"sort"
1010
"strings"
1111
"sync"
12-
13-
lru "github.com/hashicorp/golang-lru/v2"
1412
)
1513

1614
const (
@@ -41,7 +39,6 @@ type WAL struct {
4139
olderSegments map[SegmentID]*segment // older segment files, only used for read.
4240
options Options
4341
mu sync.RWMutex
44-
blockCache *lru.Cache[uint64, []byte]
4542
bytesWrite uint32
4643
renameIds []SegmentID
4744
pendingWrites [][]byte
@@ -67,9 +64,6 @@ func Open(options Options) (*WAL, error) {
6764
if !strings.HasPrefix(options.SegmentFileExt, ".") {
6865
return nil, fmt.Errorf("segment file extension must start with '.'")
6966
}
70-
if options.BlockCache > uint32(options.SegmentSize) {
71-
return nil, fmt.Errorf("BlockCache must be smaller than SegmentSize")
72-
}
7367
wal := &WAL{
7468
options: options,
7569
olderSegments: make(map[SegmentID]*segment),
@@ -81,19 +75,6 @@ func Open(options Options) (*WAL, error) {
8175
return nil, err
8276
}
8377

84-
// create the block cache if needed.
85-
if options.BlockCache > 0 {
86-
var lruSize = options.BlockCache / blockSize
87-
if options.BlockCache%blockSize != 0 {
88-
lruSize += 1
89-
}
90-
cache, err := lru.New[uint64, []byte](int(lruSize))
91-
if err != nil {
92-
return nil, err
93-
}
94-
wal.blockCache = cache
95-
}
96-
9778
// iterate the dir and open all segment files.
9879
entries, err := os.ReadDir(options.DirPath)
9980
if err != nil {
@@ -117,7 +98,7 @@ func Open(options Options) (*WAL, error) {
11798
// empty directory, just initialize a new segment file.
11899
if len(segmentIDs) == 0 {
119100
segment, err := openSegmentFile(options.DirPath, options.SegmentFileExt,
120-
initialSegmentFileID, wal.blockCache)
101+
initialSegmentFileID)
121102
if err != nil {
122103
return nil, err
123104
}
@@ -128,7 +109,7 @@ func Open(options Options) (*WAL, error) {
128109

129110
for i, segId := range segmentIDs {
130111
segment, err := openSegmentFile(options.DirPath, options.SegmentFileExt,
131-
uint32(segId), wal.blockCache)
112+
uint32(segId))
132113
if err != nil {
133114
return nil, err
134115
}
@@ -163,7 +144,7 @@ func (wal *WAL) OpenNewActiveSegment() error {
163144
}
164145
// create a new segment file and set it as the active one.
165146
segment, err := openSegmentFile(wal.options.DirPath, wal.options.SegmentFileExt,
166-
wal.activeSegment.id+1, wal.blockCache)
147+
wal.activeSegment.id+1)
167148
if err != nil {
168149
return err
169150
}
@@ -331,7 +312,7 @@ func (wal *WAL) rotateActiveSegment() error {
331312
}
332313
wal.bytesWrite = 0
333314
segment, err := openSegmentFile(wal.options.DirPath, wal.options.SegmentFileExt,
334-
wal.activeSegment.id+1, wal.blockCache)
315+
wal.activeSegment.id+1)
335316
if err != nil {
336317
return err
337318
}
@@ -440,11 +421,6 @@ func (wal *WAL) Close() error {
440421
wal.mu.Lock()
441422
defer wal.mu.Unlock()
442423

443-
// purge the block cache.
444-
if wal.blockCache != nil {
445-
wal.blockCache.Purge()
446-
}
447-
448424
// close all segment files.
449425
for _, segment := range wal.olderSegments {
450426
if err := segment.Close(); err != nil {
@@ -464,11 +440,6 @@ func (wal *WAL) Delete() error {
464440
wal.mu.Lock()
465441
defer wal.mu.Unlock()
466442

467-
// purge the block cache.
468-
if wal.blockCache != nil {
469-
wal.blockCache.Purge()
470-
}
471-
472443
// delete all segment files.
473444
for _, segment := range wal.olderSegments {
474445
if err := segment.Remove(); err != nil {

‎wal_test.go

+1-9
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
package wal
22

33
import (
4+
"github.com/stretchr/testify/assert"
45
"io"
56
"os"
67
"strings"
78
"testing"
8-
9-
"github.com/stretchr/testify/assert"
109
)
1110

1211
func destroyWAL(wal *WAL) {
@@ -22,7 +21,6 @@ func TestWAL_WriteALL(t *testing.T) {
2221
DirPath: dir,
2322
SegmentFileExt: ".SEG",
2423
SegmentSize: 32 * 1024 * 1024,
25-
BlockCache: 32 * KB * 10,
2624
}
2725
wal, err := Open(opts)
2826
assert.Nil(t, err)
@@ -41,7 +39,6 @@ func TestWAL_Write(t *testing.T) {
4139
DirPath: dir,
4240
SegmentFileExt: ".SEG",
4341
SegmentSize: 32 * 1024 * 1024,
44-
BlockCache: 32 * KB * 10,
4542
}
4643
wal, err := Open(opts)
4744
assert.Nil(t, err)
@@ -75,7 +72,6 @@ func TestWAL_Write_large(t *testing.T) {
7572
DirPath: dir,
7673
SegmentFileExt: ".SEG",
7774
SegmentSize: 32 * 1024 * 1024,
78-
BlockCache: 32 * KB * 10,
7975
}
8076
wal, err := Open(opts)
8177
assert.Nil(t, err)
@@ -143,7 +139,6 @@ func TestWAL_Reader(t *testing.T) {
143139
DirPath: dir,
144140
SegmentFileExt: ".SEG",
145141
SegmentSize: 32 * 1024 * 1024,
146-
BlockCache: 32 * KB * 10,
147142
}
148143
wal, err := Open(opts)
149144
assert.Nil(t, err)
@@ -248,7 +243,6 @@ func TestWAL_Delete(t *testing.T) {
248243
DirPath: dir,
249244
SegmentFileExt: ".SEG",
250245
SegmentSize: 32 * 1024 * 1024,
251-
BlockCache: 32 * KB * 10,
252246
}
253247
wal, err := Open(opts)
254248
assert.Nil(t, err)
@@ -270,7 +264,6 @@ func TestWAL_ReaderWithStart(t *testing.T) {
270264
DirPath: dir,
271265
SegmentFileExt: ".SEG",
272266
SegmentSize: 8 * 1024 * 1024,
273-
BlockCache: 32 * KB * 10,
274267
}
275268
wal, err := Open(opts)
276269
assert.Nil(t, err)
@@ -306,7 +299,6 @@ func TestWAL_RenameFileExt(t *testing.T) {
306299
DirPath: dir,
307300
SegmentFileExt: ".VLOG.1.temp",
308301
SegmentSize: 8 * 1024 * 1024,
309-
BlockCache: 32 * KB * 10,
310302
}
311303
wal, err := Open(opts)
312304
assert.Nil(t, err)

0 commit comments

Comments
 (0)