Skip to content

Commit

Permalink
Refactor to use constants for default values
Browse files Browse the repository at this point in the history
  • Loading branch information
@fxamacker
fxamacker committed Jul 27, 2022
1 parent f048d62 commit c5455bd
Show file tree
Hide file tree
Showing 8 changed files with 191 additions and 82 deletions.
14 changes: 11 additions & 3 deletions cmd/bootstrap/run/execution_state.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package run

import (
"math"

"github.com/rs/zerolog"

"github.com/onflow/flow-go/crypto"
Expand Down Expand Up @@ -36,19 +38,25 @@ func GenerateExecutionState(
chain flow.Chain,
bootstrapOptions ...fvm.BootstrapProcedureOption,
) (flow.StateCommitment, error) {
const (
capacity = 100
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)

metricsCollector := &metrics.NoopCollector{}

diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metricsCollector, dbDir, 100, pathfinder.PathByteSize, wal.SegmentSize)
diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metricsCollector, dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize)
if err != nil {
return flow.DummyStateCommitment, err
}

ledgerStorage, err := ledger.NewLedger(diskWal, 100, metricsCollector, zerolog.Nop(), ledger.DefaultPathFinderVersion)
ledgerStorage, err := ledger.NewLedger(diskWal, capacity, metricsCollector, zerolog.Nop(), ledger.DefaultPathFinderVersion)
if err != nil {
return flow.DummyStateCommitment, err
}

compactor, err := complete.NewCompactor(ledgerStorage, diskWal, zerolog.Nop(), 100, 1_000_000, 1)
compactor, err := complete.NewCompactor(ledgerStorage, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep)
if err != nil {
return flow.DummyStateCommitment, err
}
Expand Down
7 changes: 6 additions & 1 deletion cmd/util/cmd/exec-data-json-export/ledger_exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bufio"
"encoding/hex"
"fmt"
"math"
"os"
"path/filepath"

Expand All @@ -19,6 +20,10 @@ import (

// ExportLedger exports ledger key value pairs at the given blockID
func ExportLedger(ledgerPath string, targetstate string, outputPath string) error {
const (
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)

diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, &metrics.NoopCollector{}, ledgerPath, complete.DefaultCacheSize, pathfinder.PathByteSize, wal.SegmentSize)
if err != nil {
Expand All @@ -28,7 +33,7 @@ func ExportLedger(ledgerPath string, targetstate string, outputPath string) erro
if err != nil {
return fmt.Errorf("cannot create ledger from write-a-head logs and checkpoints: %w", err)
}
compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), complete.DefaultCacheSize, 1_000_000, 1)
compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), complete.DefaultCacheSize, checkpointDistance, checkpointsToKeep)
if err != nil {
return fmt.Errorf("cannot create compactor: %w", err)
}
Expand Down
7 changes: 6 additions & 1 deletion cmd/util/cmd/execution-state-extract/execution_state_extract.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package extract

import (
"fmt"
"math"

"github.com/rs/zerolog"

Expand Down Expand Up @@ -54,7 +55,11 @@ func extractExecutionState(
return fmt.Errorf("cannot create ledger from write-a-head logs and checkpoints: %w", err)
}

compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), complete.DefaultCacheSize, 1_000_000, 1)
const (
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)
compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), complete.DefaultCacheSize, checkpointDistance, checkpointsToKeep)
if err != nil {
return fmt.Errorf("cannot create compactor: %w", err)
}
Expand Down
14 changes: 12 additions & 2 deletions cmd/util/cmd/execution-state-extract/execution_state_extract_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package extract

import (
"crypto/rand"
"math"
"testing"

"github.com/rs/zerolog"
Expand Down Expand Up @@ -74,6 +75,11 @@ func TestExtractExecutionState(t *testing.T) {

withDirs(t, func(datadir, execdir, _ string) {

const (
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)

db := common.InitStorage(datadir)
commits := badger.NewCommits(metr, db)

Expand All @@ -84,7 +90,7 @@ func TestExtractExecutionState(t *testing.T) {
require.NoError(t, err)
f, err := complete.NewLedger(diskWal, size*10, metr, zerolog.Nop(), complete.DefaultPathFinderVersion)
require.NoError(t, err)
compactor, err := complete.NewCompactor(f, diskWal, zerolog.Nop(), uint(size), 1_000_000, 1)
compactor, err := complete.NewCompactor(f, diskWal, zerolog.Nop(), uint(size), checkpointDistance, checkpointsToKeep)
require.NoError(t, err)
<-compactor.Ready()

Expand Down Expand Up @@ -156,7 +162,11 @@ func TestExtractExecutionState(t *testing.T) {
storage, err := complete.NewLedger(diskWal, 1000, metr, zerolog.Nop(), complete.DefaultPathFinderVersion)
require.NoError(t, err)

compactor, err := complete.NewCompactor(storage, diskWal, zerolog.Nop(), uint(size), 1_000_000, 1)
const (
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)
compactor, err := complete.NewCompactor(storage, diskWal, zerolog.Nop(), uint(size), checkpointDistance, checkpointsToKeep)
require.NoError(t, err)

<-compactor.Ready()
Expand Down
12 changes: 9 additions & 3 deletions engine/testutil/nodes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"math"
"path/filepath"
"testing"
"time"
Expand Down Expand Up @@ -499,13 +500,18 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit

metricsCollector := &metrics.NoopCollector{}

diskWal, err := wal.NewDiskWAL(node.Log.With().Str("subcomponent", "wal").Logger(), nil, metricsCollector, dbDir, 100, pathfinder.PathByteSize, wal.SegmentSize)
const (
capacity = 100
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)
diskWal, err := wal.NewDiskWAL(node.Log.With().Str("subcomponent", "wal").Logger(), nil, metricsCollector, dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize)
require.NoError(t, err)

ls, err := completeLedger.NewLedger(diskWal, 100, metricsCollector, node.Log.With().Str("compontent", "ledger").Logger(), completeLedger.DefaultPathFinderVersion)
ls, err := completeLedger.NewLedger(diskWal, capacity, metricsCollector, node.Log.With().Str("compontent", "ledger").Logger(), completeLedger.DefaultPathFinderVersion)
require.NoError(t, err)

compactor, err := completeLedger.NewCompactor(ls, diskWal, zerolog.Nop(), 100, 1_000_000, 1)
compactor, err := completeLedger.NewCompactor(ls, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep)
require.NoError(t, err)

<-compactor.Ready() // Need to start compactor here because BootstrapLedger() updates ledger state.
Expand Down
106 changes: 68 additions & 38 deletions ledger/complete/ledger_benchmark_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package complete_test

import (
"math"
"math/rand"
"os"
"testing"
Expand Down Expand Up @@ -28,11 +29,16 @@ func BenchmarkStorage(b *testing.B) { benchmarkStorage(100, b) }
// BenchmarkStorage benchmarks the performance of the storage layer
func benchmarkStorage(steps int, b *testing.B) {
// assumption: 1000 key updates per collection
numInsPerStep := 1000
keyNumberOfParts := 10
keyPartMinByteSize := 1
keyPartMaxByteSize := 100
valueMaxByteSize := 32
const (
numInsPerStep = 1000
keyNumberOfParts = 10
keyPartMinByteSize = 1
keyPartMaxByteSize = 100
valueMaxByteSize = 32
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)

rand.Seed(time.Now().UnixNano())

dir, err := os.MkdirTemp("", "test-mtrie-")
Expand All @@ -47,7 +53,7 @@ func benchmarkStorage(steps int, b *testing.B) {
led, err := complete.NewLedger(diskWal, steps+1, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion)
require.NoError(b, err)

compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), uint(steps+1), 1_000_000, 1)
compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), uint(steps+1), checkpointDistance, checkpointsToKeep)
require.NoError(b, err)

<-compactor.Ready()
Expand Down Expand Up @@ -141,11 +147,17 @@ func benchmarkStorage(steps int, b *testing.B) {
// BenchmarkTrieUpdate benchmarks the performance of a trie update
func BenchmarkTrieUpdate(b *testing.B) {
// key updates per iteration
numInsPerStep := 10000
keyNumberOfParts := 10
keyPartMinByteSize := 1
keyPartMaxByteSize := 100
valueMaxByteSize := 32
const (
numInsPerStep = 10000
keyNumberOfParts = 10
keyPartMinByteSize = 1
keyPartMaxByteSize = 100
valueMaxByteSize = 32
capacity = 101
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)

rand.Seed(1)

dir, err := os.MkdirTemp("", "test-mtrie-")
Expand All @@ -154,13 +166,13 @@ func BenchmarkTrieUpdate(b *testing.B) {
b.Fatal(err)
}

diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, 101, pathfinder.PathByteSize, wal.SegmentSize)
diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize)
require.NoError(b, err)

led, err := complete.NewLedger(diskWal, 101, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion)
led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion)
require.NoError(b, err)

compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 101, 1_000_000, 1)
compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep)
require.NoError(b, err)

<-compactor.Ready()
Expand Down Expand Up @@ -193,11 +205,17 @@ func BenchmarkTrieUpdate(b *testing.B) {
// BenchmarkTrieUpdate benchmarks the performance of a trie read
func BenchmarkTrieRead(b *testing.B) {
// key updates per iteration
numInsPerStep := 10000
keyNumberOfParts := 10
keyPartMinByteSize := 1
keyPartMaxByteSize := 100
valueMaxByteSize := 32
const (
numInsPerStep = 10000
keyNumberOfParts = 10
keyPartMinByteSize = 1
keyPartMaxByteSize = 100
valueMaxByteSize = 32
capacity = 101
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)

rand.Seed(1)

dir, err := os.MkdirTemp("", "test-mtrie-")
Expand All @@ -206,13 +224,13 @@ func BenchmarkTrieRead(b *testing.B) {
b.Fatal(err)
}

diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, 101, pathfinder.PathByteSize, wal.SegmentSize)
diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize)
require.NoError(b, err)

led, err := complete.NewLedger(diskWal, 101, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion)
led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion)
require.NoError(b, err)

compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 101, 1_000_000, 1)
compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep)
require.NoError(b, err)

<-compactor.Ready()
Expand Down Expand Up @@ -254,11 +272,17 @@ func BenchmarkTrieRead(b *testing.B) {

func BenchmarkLedgerGetOneValue(b *testing.B) {
// key updates per iteration
numInsPerStep := 10000
keyNumberOfParts := 10
keyPartMinByteSize := 1
keyPartMaxByteSize := 100
valueMaxByteSize := 32
const (
numInsPerStep = 10000
keyNumberOfParts = 10
keyPartMinByteSize = 1
keyPartMaxByteSize = 100
valueMaxByteSize = 32
capacity = 101
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)

rand.Seed(1)

dir, err := os.MkdirTemp("", "test-mtrie-")
Expand All @@ -267,13 +291,13 @@ func BenchmarkLedgerGetOneValue(b *testing.B) {
b.Fatal(err)
}

diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, 101, pathfinder.PathByteSize, wal.SegmentSize)
diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize)
require.NoError(b, err)

led, err := complete.NewLedger(diskWal, 101, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion)
led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion)
require.NoError(b, err)

compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 101, 1_000_000, 1)
compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep)
require.NoError(b, err)

<-compactor.Ready()
Expand Down Expand Up @@ -332,11 +356,17 @@ func BenchmarkLedgerGetOneValue(b *testing.B) {
// BenchmarkTrieUpdate benchmarks the performance of a trie prove
func BenchmarkTrieProve(b *testing.B) {
// key updates per iteration
numInsPerStep := 10000
keyNumberOfParts := 10
keyPartMinByteSize := 1
keyPartMaxByteSize := 100
valueMaxByteSize := 32
const (
numInsPerStep = 10000
keyNumberOfParts = 10
keyPartMinByteSize = 1
keyPartMaxByteSize = 100
valueMaxByteSize = 32
capacity = 101
checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation.
checkpointsToKeep = 1
)

rand.Seed(1)

dir, err := os.MkdirTemp("", "test-mtrie-")
Expand All @@ -345,13 +375,13 @@ func BenchmarkTrieProve(b *testing.B) {
b.Fatal(err)
}

diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, 101, pathfinder.PathByteSize, wal.SegmentSize)
diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize)
require.NoError(b, err)

led, err := complete.NewLedger(diskWal, 101, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion)
led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion)
require.NoError(b, err)

compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), 101, 1_000_000, 1)
compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep)
require.NoError(b, err)

<-compactor.Ready()
Expand Down
Loading

0 comments on commit c5455bd

Please sign in to comment.