diff --git a/cmd/bootstrap/run/execution_state.go b/cmd/bootstrap/run/execution_state.go index 34ee773c392..bde57a78866 100644 --- a/cmd/bootstrap/run/execution_state.go +++ b/cmd/bootstrap/run/execution_state.go @@ -1,6 +1,8 @@ package run import ( + "math" + "github.com/rs/zerolog" "github.com/onflow/flow-go/crypto" @@ -36,19 +38,25 @@ func GenerateExecutionState( chain flow.Chain, bootstrapOptions ...fvm.BootstrapProcedureOption, ) (flow.StateCommitment, error) { + const ( + capacity = 100 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + metricsCollector := &metrics.NoopCollector{} - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metricsCollector, dbDir, 100, pathfinder.PathByteSize, wal.SegmentSize) + diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metricsCollector, dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize) if err != nil { return flow.DummyStateCommitment, err } - ledgerStorage, err := ledger.NewLedger(diskWal, 100, metricsCollector, zerolog.Nop(), ledger.DefaultPathFinderVersion) + ledgerStorage, err := ledger.NewLedger(diskWal, capacity, metricsCollector, zerolog.Nop(), ledger.DefaultPathFinderVersion) if err != nil { return flow.DummyStateCommitment, err } - compactor, err := complete.NewCompactor(ledgerStorage, diskWal, zerolog.Nop(), 100, 1_000_000, 1) + compactor, err := complete.NewCompactor(ledgerStorage, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) if err != nil { return flow.DummyStateCommitment, err } diff --git a/cmd/util/cmd/exec-data-json-export/ledger_exporter.go b/cmd/util/cmd/exec-data-json-export/ledger_exporter.go index 2e3b6b6cba6..42340d13ca6 100644 --- a/cmd/util/cmd/exec-data-json-export/ledger_exporter.go +++ b/cmd/util/cmd/exec-data-json-export/ledger_exporter.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/hex" "fmt" + "math" "os" "path/filepath" @@ -19,6 +20,10 @@ import ( // ExportLedger exports ledger key value pairs at the given blockID func ExportLedger(ledgerPath string, targetstate string, outputPath string) error { + const ( + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, &metrics.NoopCollector{}, ledgerPath, complete.DefaultCacheSize, pathfinder.PathByteSize, wal.SegmentSize) if err != nil { @@ -28,7 +33,7 @@ func ExportLedger(ledgerPath string, targetstate string, outputPath string) erro if err != nil { return fmt.Errorf("cannot create ledger from write-a-head logs and checkpoints: %w", err) } - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), complete.DefaultCacheSize, 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), complete.DefaultCacheSize, checkpointDistance, checkpointsToKeep) if err != nil { return fmt.Errorf("cannot create compactor: %w", err) } diff --git a/cmd/util/cmd/execution-state-extract/execution_state_extract.go b/cmd/util/cmd/execution-state-extract/execution_state_extract.go index 22756be42cf..678ffacadc8 100644 --- a/cmd/util/cmd/execution-state-extract/execution_state_extract.go +++ b/cmd/util/cmd/execution-state-extract/execution_state_extract.go @@ -2,6 +2,7 @@ package extract import ( "fmt" + "math" "github.com/rs/zerolog" @@ -54,7 +55,11 @@ func extractExecutionState( return fmt.Errorf("cannot create ledger from write-a-head logs and checkpoints: %w", err) } - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), complete.DefaultCacheSize, 1_000_000, 1) + const ( + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), complete.DefaultCacheSize, checkpointDistance, checkpointsToKeep) if err != nil { return fmt.Errorf("cannot create compactor: %w", err) } diff --git a/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go b/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go index 61b882c3b6f..06b2832f72b 100644 --- a/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go +++ b/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go @@ -2,6 +2,7 @@ package extract import ( "crypto/rand" + "math" "testing" "github.com/rs/zerolog" @@ -74,6 +75,11 @@ func TestExtractExecutionState(t *testing.T) { withDirs(t, func(datadir, execdir, _ string) { + const ( + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + db := common.InitStorage(datadir) commits := badger.NewCommits(metr, db) @@ -84,7 +90,7 @@ func TestExtractExecutionState(t *testing.T) { require.NoError(t, err) f, err := complete.NewLedger(diskWal, size*10, metr, zerolog.Nop(), complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor, err := complete.NewCompactor(f, diskWal, zerolog.Nop(), uint(size), 1_000_000, 1) + compactor, err := complete.NewCompactor(f, diskWal, zerolog.Nop(), uint(size), checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor.Ready() @@ -156,7 +162,11 @@ func TestExtractExecutionState(t *testing.T) { storage, err := complete.NewLedger(diskWal, 1000, metr, zerolog.Nop(), complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor, err := complete.NewCompactor(storage, diskWal, zerolog.Nop(), uint(size), 1_000_000, 1) + const ( + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + compactor, err := complete.NewCompactor(storage, diskWal, zerolog.Nop(), uint(size), checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor.Ready() diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 3f7b2144a01..8aa39aa8f18 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "math" "path/filepath" "testing" "time" @@ -499,13 +500,18 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit metricsCollector := &metrics.NoopCollector{} - diskWal, err := wal.NewDiskWAL(node.Log.With().Str("subcomponent", "wal").Logger(), nil, metricsCollector, dbDir, 100, pathfinder.PathByteSize, wal.SegmentSize) + const ( + capacity = 100 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + diskWal, err := wal.NewDiskWAL(node.Log.With().Str("subcomponent", "wal").Logger(), nil, metricsCollector, dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(t, err) - ls, err := completeLedger.NewLedger(diskWal, 100, metricsCollector, node.Log.With().Str("compontent", "ledger").Logger(), completeLedger.DefaultPathFinderVersion) + ls, err := completeLedger.NewLedger(diskWal, capacity, metricsCollector, node.Log.With().Str("compontent", "ledger").Logger(), completeLedger.DefaultPathFinderVersion) require.NoError(t, err) - compactor, err := completeLedger.NewCompactor(ls, diskWal, zerolog.Nop(), 100, 1_000_000, 1) + compactor, err := completeLedger.NewCompactor(ls, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor.Ready() // Need to start compactor here because BootstrapLedger() updates ledger state. diff --git a/ledger/complete/ledger_benchmark_test.go b/ledger/complete/ledger_benchmark_test.go index 52c0ce0c55a..20ba93480e8 100644 --- a/ledger/complete/ledger_benchmark_test.go +++ b/ledger/complete/ledger_benchmark_test.go @@ -1,6 +1,7 @@ package complete_test import ( + "math" "math/rand" "os" "testing" @@ -28,11 +29,16 @@ func BenchmarkStorage(b *testing.B) { benchmarkStorage(100, b) } // BenchmarkStorage benchmarks the performance of the storage layer func benchmarkStorage(steps int, b *testing.B) { // assumption: 1000 key updates per collection - numInsPerStep := 1000 - keyNumberOfParts := 10 - keyPartMinByteSize := 1 - keyPartMaxByteSize := 100 - valueMaxByteSize := 32 + const ( + numInsPerStep = 1000 + keyNumberOfParts = 10 + keyPartMinByteSize = 1 + keyPartMaxByteSize = 100 + valueMaxByteSize = 32 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + rand.Seed(time.Now().UnixNano()) dir, err := os.MkdirTemp("", "test-mtrie-") @@ -47,7 +53,7 @@ func benchmarkStorage(steps int, b *testing.B) { led, err := complete.NewLedger(diskWal, steps+1, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(b, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), uint(steps+1), 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), uint(steps+1), checkpointDistance, checkpointsToKeep) require.NoError(b, err) <-compactor.Ready() @@ -141,11 +147,17 @@ func benchmarkStorage(steps int, b *testing.B) { // BenchmarkTrieUpdate benchmarks the performance of a trie update func BenchmarkTrieUpdate(b *testing.B) { // key updates per iteration - numInsPerStep := 10000 - keyNumberOfParts := 10 - keyPartMinByteSize := 1 - keyPartMaxByteSize := 100 - valueMaxByteSize := 32 + const ( + numInsPerStep = 10000 + keyNumberOfParts = 10 + keyPartMinByteSize = 1 + keyPartMaxByteSize = 100 + valueMaxByteSize = 32 + capacity = 101 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + rand.Seed(1) dir, err := os.MkdirTemp("", "test-mtrie-") @@ -154,13 +166,13 @@ func BenchmarkTrieUpdate(b *testing.B) { b.Fatal(err) } - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, 101, pathfinder.PathByteSize, wal.SegmentSize) + diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(b, err) - led, err := complete.NewLedger(diskWal, 101, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(b, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 101, 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(b, err) <-compactor.Ready() @@ -193,11 +205,17 @@ func BenchmarkTrieUpdate(b *testing.B) { // BenchmarkTrieUpdate benchmarks the performance of a trie read func BenchmarkTrieRead(b *testing.B) { // key updates per iteration - numInsPerStep := 10000 - keyNumberOfParts := 10 - keyPartMinByteSize := 1 - keyPartMaxByteSize := 100 - valueMaxByteSize := 32 + const ( + numInsPerStep = 10000 + keyNumberOfParts = 10 + keyPartMinByteSize = 1 + keyPartMaxByteSize = 100 + valueMaxByteSize = 32 + capacity = 101 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + rand.Seed(1) dir, err := os.MkdirTemp("", "test-mtrie-") @@ -206,13 +224,13 @@ func BenchmarkTrieRead(b *testing.B) { b.Fatal(err) } - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, 101, pathfinder.PathByteSize, wal.SegmentSize) + diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(b, err) - led, err := complete.NewLedger(diskWal, 101, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(b, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 101, 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(b, err) <-compactor.Ready() @@ -254,11 +272,17 @@ func BenchmarkTrieRead(b *testing.B) { func BenchmarkLedgerGetOneValue(b *testing.B) { // key updates per iteration - numInsPerStep := 10000 - keyNumberOfParts := 10 - keyPartMinByteSize := 1 - keyPartMaxByteSize := 100 - valueMaxByteSize := 32 + const ( + numInsPerStep = 10000 + keyNumberOfParts = 10 + keyPartMinByteSize = 1 + keyPartMaxByteSize = 100 + valueMaxByteSize = 32 + capacity = 101 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + rand.Seed(1) dir, err := os.MkdirTemp("", "test-mtrie-") @@ -267,13 +291,13 @@ func BenchmarkLedgerGetOneValue(b *testing.B) { b.Fatal(err) } - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, 101, pathfinder.PathByteSize, wal.SegmentSize) + diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(b, err) - led, err := complete.NewLedger(diskWal, 101, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(b, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 101, 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(b, err) <-compactor.Ready() @@ -332,11 +356,17 @@ func BenchmarkLedgerGetOneValue(b *testing.B) { // BenchmarkTrieUpdate benchmarks the performance of a trie prove func BenchmarkTrieProve(b *testing.B) { // key updates per iteration - numInsPerStep := 10000 - keyNumberOfParts := 10 - keyPartMinByteSize := 1 - keyPartMaxByteSize := 100 - valueMaxByteSize := 32 + const ( + numInsPerStep = 10000 + keyNumberOfParts = 10 + keyPartMinByteSize = 1 + keyPartMaxByteSize = 100 + valueMaxByteSize = 32 + capacity = 101 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + rand.Seed(1) dir, err := os.MkdirTemp("", "test-mtrie-") @@ -345,13 +375,13 @@ func BenchmarkTrieProve(b *testing.B) { b.Fatal(err) } - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, 101, pathfinder.PathByteSize, wal.SegmentSize) + diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(b, err) - led, err := complete.NewLedger(diskWal, 101, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(b, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 101, 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(b, err) <-compactor.Ready() diff --git a/ledger/complete/ledger_test.go b/ledger/complete/ledger_test.go index 7c30ccb08d4..408c61e6a6d 100644 --- a/ledger/complete/ledger_test.go +++ b/ledger/complete/ledger_test.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "math" "math/rand" "testing" "time" @@ -490,12 +491,17 @@ func TestLedger_Proof(t *testing.T) { } func Test_WAL(t *testing.T) { - numInsPerStep := 2 - keyNumberOfParts := 10 - keyPartMinByteSize := 1 - keyPartMaxByteSize := 100 - valueMaxByteSize := 2 << 16 //16kB - size := 10 + const ( + numInsPerStep = 2 + keyNumberOfParts = 10 + keyPartMinByteSize = 1 + keyPartMaxByteSize = 100 + valueMaxByteSize = 2 << 16 //16kB + size = 10 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + metricsCollector := &metrics.NoopCollector{} logger := zerolog.Logger{} @@ -508,7 +514,7 @@ func Test_WAL(t *testing.T) { led, err := complete.NewLedger(diskWal, size, metricsCollector, logger, complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), uint(size), 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), size, checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor.Ready() @@ -546,7 +552,7 @@ func Test_WAL(t *testing.T) { led2, err := complete.NewLedger(diskWal2, size+10, metricsCollector, logger, complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), uint(size), 1_000_000, 1) + compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), uint(size), checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor2.Ready() @@ -585,6 +591,11 @@ func Test_WAL(t *testing.T) { } func TestLedgerFunctionality(t *testing.T) { + const ( + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + rand.Seed(time.Now().UnixNano()) // You can manually increase this for more coverage experimentRep := 2 @@ -608,7 +619,7 @@ func TestLedgerFunctionality(t *testing.T) { require.NoError(t, err) led, err := complete.NewLedger(diskWal, activeTries, metricsCollector, logger, complete.DefaultPathFinderVersion) assert.NoError(t, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), uint(activeTries), 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), uint(activeTries), checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor.Ready() @@ -709,11 +720,17 @@ func Test_ExportCheckpointAt(t *testing.T) { unittest.RunWithTempDir(t, func(dbDir string) { unittest.RunWithTempDir(t, func(dir2 string) { - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dbDir, 100, pathfinder.PathByteSize, wal.SegmentSize) + const ( + capacity = 100 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + + diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(t, err) - led, err := complete.NewLedger(diskWal, 100, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 100, 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor.Ready() @@ -728,11 +745,11 @@ func Test_ExportCheckpointAt(t *testing.T) { require.NoError(t, err) assert.Equal(t, newState, state) - diskWal2, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir2, 100, pathfinder.PathByteSize, wal.SegmentSize) + diskWal2, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir2, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(t, err) - led2, err := complete.NewLedger(diskWal2, 100, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led2, err := complete.NewLedger(diskWal2, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), 100, 1_000_000, 1) + compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor2.Ready() @@ -761,11 +778,17 @@ func Test_ExportCheckpointAt(t *testing.T) { unittest.RunWithTempDir(t, func(dbDir string) { unittest.RunWithTempDir(t, func(dir2 string) { - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dbDir, 100, pathfinder.PathByteSize, wal.SegmentSize) + const ( + capacity = 100 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + + diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(t, err) - led, err := complete.NewLedger(diskWal, 100, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 100, 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor.Ready() @@ -779,11 +802,11 @@ func Test_ExportCheckpointAt(t *testing.T) { newState, err := led.ExportCheckpointAt(state, []ledger.Migration{migrationByValue}, []ledger.Reporter{}, []ledger.Reporter{}, complete.DefaultPathFinderVersion, dir2, "root.checkpoint") require.NoError(t, err) - diskWal2, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir2, 100, pathfinder.PathByteSize, wal.SegmentSize) + diskWal2, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir2, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(t, err) - led2, err := complete.NewLedger(diskWal2, 100, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led2, err := complete.NewLedger(diskWal2, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), 100, 1_000_000, 1) + compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor2.Ready() @@ -811,11 +834,17 @@ func Test_ExportCheckpointAt(t *testing.T) { unittest.RunWithTempDir(t, func(dbDir string) { unittest.RunWithTempDir(t, func(dir2 string) { - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dbDir, 100, pathfinder.PathByteSize, wal.SegmentSize) + const ( + capacity = 100 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + + diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(t, err) - led, err := complete.NewLedger(diskWal, 100, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 100, 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor.Ready() @@ -829,11 +858,11 @@ func Test_ExportCheckpointAt(t *testing.T) { newState, err := led.ExportCheckpointAt(state, []ledger.Migration{migrationByKey}, []ledger.Reporter{}, []ledger.Reporter{}, complete.DefaultPathFinderVersion, dir2, "root.checkpoint") require.NoError(t, err) - diskWal2, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir2, 100, pathfinder.PathByteSize, wal.SegmentSize) + diskWal2, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir2, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(t, err) - led2, err := complete.NewLedger(diskWal2, 100, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led2, err := complete.NewLedger(diskWal2, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), 100, 1_000_000, 1) + compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor2.Ready() @@ -857,11 +886,18 @@ func Test_ExportCheckpointAt(t *testing.T) { func TestWALUpdateFailuresBubbleUp(t *testing.T) { unittest.RunWithTempDir(t, func(dir string) { + + const ( + capacity = 100 + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + theError := fmt.Errorf("error error") metricsCollector := &metrics.NoopCollector{} - diskWAL, err := wal.NewDiskWAL(zerolog.Nop(), nil, metricsCollector, dir, 100, pathfinder.PathByteSize, wal.SegmentSize) + diskWAL, err := wal.NewDiskWAL(zerolog.Nop(), nil, metricsCollector, dir, capacity, pathfinder.PathByteSize, wal.SegmentSize) require.NoError(t, err) w := &CustomUpdateWAL{ @@ -871,10 +907,10 @@ func TestWALUpdateFailuresBubbleUp(t *testing.T) { }, } - led, err := complete.NewLedger(w, 100, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) + led, err := complete.NewLedger(w, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor, err := complete.NewCompactor(led, w, zerolog.Nop(), 100, 1_000_000, 1) + compactor, err := complete.NewCompactor(led, w, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor.Ready() diff --git a/ledger/complete/wal/checkpointer_test.go b/ledger/complete/wal/checkpointer_test.go index af35f25a8cb..23c60be0b59 100644 --- a/ledger/complete/wal/checkpointer_test.go +++ b/ledger/complete/wal/checkpointer_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "io/ioutil" + "math" "math/rand" "os" "path" @@ -30,31 +31,39 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -var ( +const ( numInsPerStep = 2 keyNumberOfParts = 10 keyPartMinByteSize = 1 keyPartMaxByteSize = 100 valueMaxByteSize = 2 << 16 //16kB size = 10 - metricsCollector = &metrics.NoopCollector{} - logger = zerolog.Logger{} segmentSize = 32 * 1024 pathByteSize = 32 pathFinderVersion = uint8(complete.DefaultPathFinderVersion) ) +var ( + logger = zerolog.Logger{} + metricsCollector = &metrics.NoopCollector{} +) + func Test_WAL(t *testing.T) { unittest.RunWithTempDir(t, func(dir string) { + const ( + checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. + checkpointsToKeep = 1 + ) + diskWal, err := realWAL.NewDiskWAL(zerolog.Nop(), nil, metricsCollector, dir, size, pathfinder.PathByteSize, realWAL.SegmentSize) require.NoError(t, err) led, err := complete.NewLedger(diskWal, size*10, metricsCollector, logger, complete.DefaultPathFinderVersion) require.NoError(t, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), uint(size), 1_000_000, 1) + compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), size, checkpointDistance, checkpointsToKeep) require.NoError(t, err) <-compactor.Ready()