Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion x/blockdb/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ BlockDB is a specialized database optimized for blockchain blocks.
- **Configurable Durability**: Optional `syncToDisk` mode guarantees immediate recoverability
- **Automatic Recovery**: Detects and recovers unindexed blocks after unclean shutdowns
- **Block Compression**: zstd compression for block data
- **In-Memory Cache**: LRU cache for recently accessed blocks

## Design

Expand Down Expand Up @@ -167,7 +168,6 @@ if err != nil {

## TODO

- Implement a block cache for recently accessed blocks
- Use a buffered pool to avoid allocations on reads and writes
- Add performance benchmarks
- Consider supporting missing data files (currently we error if any data files are missing)
85 changes: 85 additions & 0 deletions x/blockdb/cache_db.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package blockdb

import (
"slices"

"go.uber.org/zap"

"github.com/ava-labs/avalanchego/cache/lru"
"github.com/ava-labs/avalanchego/database"
)

var _ database.HeightIndex = (*cacheDB)(nil)

// cacheDB caches data from the underlying [Database].
//
// Operations (Get, Has, Put) are not atomic with the underlying database.
// Concurrent writes to the same height can result in cache inconsistencies where
// the cache and database contain different values. This limitation is acceptable
// because concurrent writes to the same height are not an intended use case.
Copy link
Contributor

@rrazvan1 rrazvan1 Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
// because concurrent writes to the same height are not an intended use case.
// because concurrent access to the same height is not an intended use case.

Copy link
Contributor Author

@DracoLi DracoLi Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

concurrent Get/Has to the same height is fine and expected. its the writes to the same height that is not intended usage and will result in access inconsistencies from cached Has and Get

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then it sounds wrong to me:

Concurrent access to the same height can result in cache inconsistencies [...] is acceptable because concurrent writes to the same height are not an intended use case.

Copy link
Contributor Author

@DracoLi DracoLi Nov 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

reverted to the original comment. 9792f38
I think both are right depending on how you interpret it but I think we can just call out the writes.

type cacheDB struct {
db *Database
cache *lru.Cache[BlockHeight, BlockData]
}

func newCacheDB(db *Database, size uint16) *cacheDB {
return &cacheDB{
db: db,
cache: lru.NewCache[BlockHeight, BlockData](int(size)),
}
}

func (c *cacheDB) Get(height BlockHeight) (BlockData, error) {
c.db.closeMu.RLock()
defer c.db.closeMu.RUnlock()

if c.db.closed {
c.db.log.Error("Failed Get: database closed", zap.Uint64("height", height))
return nil, database.ErrClosed
}

if cached, ok := c.cache.Get(height); ok {
return slices.Clone(cached), nil
}
data, err := c.db.getWithoutLock(height)
if err != nil {
return nil, err
}
c.cache.Put(height, slices.Clone(data))
return data, nil
}

func (c *cacheDB) Put(height BlockHeight, data BlockData) error {
if err := c.db.Put(height, data); err != nil {
return err
}

c.cache.Put(height, slices.Clone(data))
return nil
}

func (c *cacheDB) Has(height BlockHeight) (bool, error) {
c.db.closeMu.RLock()
defer c.db.closeMu.RUnlock()

if c.db.closed {
c.db.log.Error("Failed Has: database closed", zap.Uint64("height", height))
return false, database.ErrClosed
}

if _, ok := c.cache.Get(height); ok {
return true, nil
}
return c.db.hasWithoutLock(height)
}

func (c *cacheDB) Close() error {
if err := c.db.Close(); err != nil {
return err
}
c.cache.Flush()
return nil
}
118 changes: 118 additions & 0 deletions x/blockdb/cache_db_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package blockdb

import (
"slices"
"testing"

"github.com/stretchr/testify/require"
)

func TestCacheOnMiss(t *testing.T) {
db := newCacheDatabase(t, DefaultConfig())
height := uint64(20)
block := randomBlock(t)
require.NoError(t, db.Put(height, block))

// Evict the entry from cache to simulate a cache miss
db.cache.Evict(height)

// Read the block - should populate the cache on cache miss
_, err := db.Get(height)
require.NoError(t, err)

_, ok := db.cache.Get(height)
require.True(t, ok)
}

func TestCacheGet(t *testing.T) {
db := newCacheDatabase(t, DefaultConfig())
height := uint64(30)
block := randomBlock(t)

// Populate cache directly without writing to database
db.cache.Put(height, block)

// Get should return the block from cache
data, err := db.Get(height)
require.NoError(t, err)
require.Equal(t, block, data)
}

func TestCacheHas(t *testing.T) {
db := newCacheDatabase(t, DefaultConfig())
height := uint64(40)
block := randomBlock(t)

// Populate cache directly without writing to database
db.cache.Put(height, block)

// Has should return true from cache even though block is not in database
has, err := db.Has(height)
require.NoError(t, err)
require.True(t, has)
}

func TestCachePutStoresClone(t *testing.T) {
db := newCacheDatabase(t, DefaultConfig())
height := uint64(40)
block := randomBlock(t)
clone := slices.Clone(block)
require.NoError(t, db.Put(height, clone))

// Modify the original block after Put
clone[0] = 99

// Cache should have the original unmodified data
cached, ok := db.cache.Get(height)
require.True(t, ok)
require.Equal(t, block, cached)
}

func TestCacheGetReturnsClone(t *testing.T) {
db := newCacheDatabase(t, DefaultConfig())
height := uint64(50)
block := randomBlock(t)
require.NoError(t, db.Put(height, block))

// Get the block and modify the returned data
data, err := db.Get(height)
require.NoError(t, err)
data[0] = 99

// Cache should still have the original unmodified data
cached, ok := db.cache.Get(height)
require.True(t, ok)
require.Equal(t, block, cached)

// Second Get should also return original data
data, err = db.Get(height)
require.NoError(t, err)
require.Equal(t, block, data)
}

func TestCachePutOverridesSameHeight(t *testing.T) {
db := newCacheDatabase(t, DefaultConfig())
height := uint64(60)
b1 := randomBlock(t)
require.NoError(t, db.Put(height, b1))

// Verify first block is in cache
cached, ok := db.cache.Get(height)
require.True(t, ok)
require.Equal(t, b1, cached)

// Put second block at same height and verify it overrides the first one
b2 := randomBlock(t)
require.NoError(t, db.Put(height, b2))
cached, ok = db.cache.Get(height)
require.True(t, ok)
require.Equal(t, b2, cached)

// Get should also return the new block
data, err := db.Get(height)
require.NoError(t, err)
require.Equal(t, b2, data)
}
13 changes: 13 additions & 0 deletions x/blockdb/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ const DefaultMaxDataFileSize = 500 * 1024 * 1024 * 1024
// DefaultMaxDataFiles is the default maximum number of data files descriptors cached.
const DefaultMaxDataFiles = 10

// DefaultBlockCacheSize is the default size of the block cache.
const DefaultBlockCacheSize uint16 = 256

// DatabaseConfig contains configuration parameters for BlockDB.
type DatabaseConfig struct {
// IndexDir is the directory where the index file is stored.
Expand All @@ -28,6 +31,9 @@ type DatabaseConfig struct {
// MaxDataFiles is the maximum number of data files descriptors cached.
MaxDataFiles int

// BlockCacheSize is the size of the block cache (default: 256).
BlockCacheSize uint16

// CheckpointInterval defines how frequently (in blocks) the index file header is updated (default: 1024).
CheckpointInterval uint64

Expand All @@ -43,6 +49,7 @@ func DefaultConfig() DatabaseConfig {
MinimumHeight: 0,
MaxDataFileSize: DefaultMaxDataFileSize,
MaxDataFiles: DefaultMaxDataFiles,
BlockCacheSize: DefaultBlockCacheSize,
CheckpointInterval: 1024,
SyncToDisk: true,
}
Expand Down Expand Up @@ -91,6 +98,12 @@ func (c DatabaseConfig) WithMaxDataFiles(maxFiles int) DatabaseConfig {
return c
}

// WithBlockCacheSize returns a copy of the config with BlockCacheSize set to the given value.
func (c DatabaseConfig) WithBlockCacheSize(size uint16) DatabaseConfig {
c.BlockCacheSize = size
return c
}

// WithCheckpointInterval returns a copy of the config with CheckpointInterval set to the given value.
func (c DatabaseConfig) WithCheckpointInterval(interval uint64) DatabaseConfig {
c.CheckpointInterval = interval
Expand Down
34 changes: 24 additions & 10 deletions x/blockdb/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ type Database struct {
// Parameters:
// - config: Configuration parameters
// - log: Logger instance for structured logging
func New(config DatabaseConfig, log logging.Logger) (*Database, error) {
func New(config DatabaseConfig, log logging.Logger) (database.HeightIndex, error) {
if err := config.Validate(); err != nil {
return nil, err
}
Expand Down Expand Up @@ -231,6 +231,7 @@ func New(config DatabaseConfig, log logging.Logger) (*Database, error) {
zap.String("dataDir", config.DataDir),
zap.Uint64("maxDataFileSize", config.MaxDataFileSize),
zap.Int("maxDataFiles", config.MaxDataFiles),
zap.Uint16("blockCacheSize", config.BlockCacheSize),
)

if err := s.openAndInitializeIndex(); err != nil {
Expand All @@ -256,6 +257,9 @@ func New(config DatabaseConfig, log logging.Logger) (*Database, error) {
zap.Uint64("maxBlockHeight", maxHeight),
)

if config.BlockCacheSize > 0 {
return newCacheDB(s, config.BlockCacheSize), nil
}
return s, nil
}

Expand Down Expand Up @@ -286,9 +290,7 @@ func (s *Database) Put(height BlockHeight, block BlockData) error {
defer s.closeMu.RUnlock()

if s.closed {
s.log.Error("Failed to write block: database is closed",
zap.Uint64("height", height),
)
s.log.Error("Failed Put: database closed", zap.Uint64("height", height))
return database.ErrClosed
}

Expand Down Expand Up @@ -385,12 +387,6 @@ func (s *Database) Put(height BlockHeight, block BlockData) error {
// It returns database.ErrNotFound if the block does not exist.
func (s *Database) readBlockIndex(height BlockHeight) (indexEntry, error) {
var entry indexEntry
if s.closed {
s.log.Error("Failed to read block index: database is closed",
zap.Uint64("height", height),
)
return entry, database.ErrClosed
}

// Skip the index entry read if we know the block is past the max height.
maxHeight := s.maxBlockHeight.Load()
Expand Down Expand Up @@ -436,6 +432,15 @@ func (s *Database) Get(height BlockHeight) (BlockData, error) {
s.closeMu.RLock()
defer s.closeMu.RUnlock()

if s.closed {
s.log.Error("Failed Get: database closed", zap.Uint64("height", height))
return nil, database.ErrClosed
}

return s.getWithoutLock(height)
}

func (s *Database) getWithoutLock(height BlockHeight) (BlockData, error) {
indexEntry, err := s.readBlockIndex(height)
if err != nil {
return nil, err
Expand Down Expand Up @@ -494,6 +499,15 @@ func (s *Database) Has(height BlockHeight) (bool, error) {
s.closeMu.RLock()
defer s.closeMu.RUnlock()

if s.closed {
s.log.Error("Failed Has: database closed", zap.Uint64("height", height))
return false, database.ErrClosed
}

return s.hasWithoutLock(height)
}

func (s *Database) hasWithoutLock(height BlockHeight) (bool, error) {
_, err := s.readBlockIndex(height)
if err != nil {
if errors.Is(err, database.ErrNotFound) || errors.Is(err, ErrInvalidBlockHeight) {
Expand Down
Loading
Loading