Separate snow package from snow vm refactor

chainstore/chain_store.go

@@ -0,0 +1,202 @@
+// Copyright (C) 2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package chainstore
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/ava-labs/avalanchego/database"
+	"github.com/ava-labs/avalanchego/ids"
+	"github.com/ava-labs/avalanchego/utils/logging"
+	"go.uber.org/zap"
+
+	"github.com/ava-labs/hypersdk/consts"
+
+	hcontext "github.com/ava-labs/hypersdk/context"
+)
+
+const (
+	blockPrefix         byte = 0x0 // TODO: move to flat files (https://github.com/ava-labs/hypersdk/issues/553)
+	blockIDHeightPrefix byte = 0x1 // ID -> Height
+	blockHeightIDPrefix byte = 0x2 // Height -> ID (don't always need full block from disk)
+	lastAcceptedByte    byte = 0x3 // lastAcceptedByte -> lastAcceptedHeight
+)
+
+type Config struct {
+	AcceptedBlockWindow             uint64 `json:"acceptedBlockWindow"`
+	BlockCompactionAverageFrequency int    `json:"blockCompactionFrequency"`
+}
+
+func NewDefaultConfig() Config {
+	return Config{
+		AcceptedBlockWindow:             50_000, // ~3.5hr with 250ms block time (100GB at 2MB)
+		BlockCompactionAverageFrequency: 32,     // 64 MB of deletion if 2 MB blocks
+	}
+}
+
+type ChainStore[T Block] struct {
+	config  Config
+	metrics *metrics
+	log     logging.Logger
+	db      database.Database
+	parser  Parser[T]
+}
+
+type Block interface {
+	ID() ids.ID
+	Height() uint64
+	Bytes() []byte
+}
+
+type Parser[T Block] interface {
+	ParseBlock(context.Context, []byte) (T, error)
+}
+
+func New[T Block](
+	hctx *hcontext.Context,
+	namespace string,
+	parser Parser[T],
+	db database.Database,
+) (*ChainStore[T], error) {
+	registry, err := hctx.MakeRegistry(namespace)
+	if err != nil {
+		return nil, err
+	}
+	metrics, err := newMetrics(registry)
+	if err != nil {
+		return nil, err
+	}
+	config, err := hcontext.GetConfigFromContext(hctx, namespace, NewDefaultConfig())
+	if err != nil {
+		return nil, err
+	}
+
+	return &ChainStore[T]{
+		config:  config,
+		metrics: metrics,
+		log:     hctx.Log(),
+		db:      db,
+		parser:  parser,
+	}, nil
+}
+
+func (c *ChainStore[T]) GetLastAcceptedHeight(_ context.Context) (uint64, error) {
+	lastAcceptedHeightBytes, err := c.db.Get([]byte{lastAcceptedByte})
+	if err != nil {
+		return 0, err
+	}
+	return database.ParseUInt64(lastAcceptedHeightBytes)
+}
+
+func (c *ChainStore[T]) UpdateLastAccepted(_ context.Context, blk T) error {
+	batch := c.db.NewBatch()
+
+	var (
+		blkID    = blk.ID()
+		height   = blk.Height()
+		blkBytes = blk.Bytes()
+	)
+	heightBytes := binary.BigEndian.AppendUint64(nil, height)
+	err := errors.Join(
+		batch.Put([]byte{lastAcceptedByte}, heightBytes),
+		batch.Put(PrefixBlockIDHeightKey(blkID), heightBytes),
+		batch.Put(PrefixBlockHeightIDKey(blk.Height()), blkID[:]),
+		batch.Put(PrefixBlockKey(height), blkBytes),
+	)
+	if err != nil {
+		return err
+	}
+	expiryHeight := height - c.config.AcceptedBlockWindow
+	var expired bool
+	if expiryHeight > 0 && expiryHeight < height { // ensure we don't free genesis
+		if err := batch.Delete(PrefixBlockKey(expiryHeight)); err != nil {
+			return err
+		}
+		blkID, err := c.db.Get(PrefixBlockHeightIDKey(expiryHeight))
+		if err != nil {
+			return fmt.Errorf("unable to fetch blockID at height %d: %w", expiryHeight, err)
+		}
+		if err := batch.Delete(PrefixBlockIDHeightKey(ids.ID(blkID))); err != nil {
+			return err
+		}
+		if err := batch.Delete(PrefixBlockHeightIDKey(expiryHeight)); err != nil {
+			return err
+		}
+		expired = true
+		c.metrics.deletedBlocks.Inc()
+	}
+	//nolint:gosec
+	if expired && rand.Intn(c.config.BlockCompactionAverageFrequency) == 0 {
+		go func() {
+			start := time.Now()
+			if err := c.db.Compact([]byte{blockPrefix}, PrefixBlockKey(expiryHeight)); err != nil {
+				c.log.Error("failed to compact block store", zap.Error(err))
+				return
+			}
+			c.log.Info("compacted disk blocks", zap.Uint64("end", expiryHeight), zap.Duration("t", time.Since(start)))
+		}()
+	}
+
+	return batch.Write()
+}
+
+func (c *ChainStore[T]) GetBlock(ctx context.Context, blkID ids.ID) (T, error) {
+	var emptyT T
+	height, err := c.GetBlockIDHeight(ctx, blkID)
+	if err != nil {
+		return emptyT, err
+	}
+	return c.GetBlockByHeight(ctx, height)
+}
+
+func (c *ChainStore[T]) GetBlockIDAtHeight(_ context.Context, blkHeight uint64) (ids.ID, error) {
+	blkIDBytes, err := c.db.Get(PrefixBlockHeightIDKey(blkHeight))
+	if err != nil {
+		return ids.Empty, err
+	}
+	return ids.ID(blkIDBytes), nil
+}
+
+func (c *ChainStore[T]) GetBlockIDHeight(_ context.Context, blkID ids.ID) (uint64, error) {
+	blkHeightBytes, err := c.db.Get(PrefixBlockIDHeightKey(blkID))
+	if err != nil {
+		return 0, err
+	}
+	return database.ParseUInt64(blkHeightBytes)
+}
+
+func (c *ChainStore[T]) GetBlockByHeight(ctx context.Context, blkHeight uint64) (T, error) {
+	var emptyT T
+	blkBytes, err := c.db.Get(PrefixBlockKey(blkHeight))
+	if err != nil {
+		return emptyT, err
+	}
+	return c.parser.ParseBlock(ctx, blkBytes)
+}
+
+func PrefixBlockKey(height uint64) []byte {
+	k := make([]byte, 1+consts.Uint64Len)
+	k[0] = blockPrefix
+	binary.BigEndian.PutUint64(k[1:], height)
+	return k
+}
+
+func PrefixBlockIDHeightKey(id ids.ID) []byte {
+	k := make([]byte, 1+ids.IDLen)
+	k[0] = blockIDHeightPrefix
+	copy(k[1:], id[:])
+	return k
+}
+
+func PrefixBlockHeightIDKey(height uint64) []byte {
+	k := make([]byte, 1+consts.Uint64Len)
+	k[0] = blockHeightIDPrefix
+	binary.BigEndian.PutUint64(k[1:], height)
+	return k
+}

chainstore/metrics.go

@@ -0,0 +1,26 @@
+// Copyright (C) 2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package chainstore
+
+import "github.com/prometheus/client_golang/prometheus"
+
+type metrics struct {
+	deletedBlocks prometheus.Counter
+}
+
+func newMetrics(registry prometheus.Registerer) (*metrics, error) {
+	m := &metrics{
+		deletedBlocks: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: "chainstore",
+			Name:      "deleted_blocks",
+			Help:      "Number of blocks deleted from the chain",
+		}),
+	}
+
+	if err := registry.Register(m.deletedBlocks); err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}

context/config.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2024, Ava Labs, Inv. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package context
+
+import "encoding/json"
+
+type Config map[string]json.RawMessage
+
+func NewConfig(b []byte) (Config, error) {
+	c := Config{}
+	if len(b) > 0 {
+		if err := json.Unmarshal(b, &c); err != nil {
+			return nil, err
+		}
+	}
+	return c, nil
+}
+
+func (c Config) Get(key string) ([]byte, bool) {
+	if val, ok := c[key]; ok {
+		return val, true
+	}
+	return nil, false
+}
+
+func GetConfig[T any](c Config, key string, defaultConfig T) (T, error) {
+	val, ok := c[key]
+	if !ok {
+		return defaultConfig, nil
+	}
+
+	var emptyConfig T
+	if err := json.Unmarshal(val, &defaultConfig); err != nil {
+		return emptyConfig, err
+	}
+	return defaultConfig, nil
+}

context/config_test.go

@@ -0,0 +1,79 @@
+// Copyright (C) 2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package context
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type testConfig struct {
+	TxFee  uint64 `json:"txFee"`
+	MinFee uint64 `json:"minFee"`
+}
+
+func TestConfigC(t *testing.T) {
+	type test struct {
+		name          string
+		providedStr   string
+		defaultConfig testConfig
+		wantConfig    testConfig
+	}
+	for _, test := range []test{
+		{
+			name:          "default want non-zero values",
+			providedStr:   "",
+			defaultConfig: testConfig{TxFee: 100},
+			wantConfig:    testConfig{TxFee: 100},
+		},
+		{
+			name:          "default want zero values",
+			providedStr:   "",
+			defaultConfig: testConfig{},
+			wantConfig:    testConfig{},
+		},
+		{
+			name: "override default with zero values",
+			providedStr: `{
+				"test": {
+					"txFee": 0,
+					"minFee": 0
+				}
+			}`,
+			defaultConfig: testConfig{TxFee: 100, MinFee: 100},
+			wantConfig:    testConfig{TxFee: 0, MinFee: 0},
+		},
+		{
+			name: "override non-zero defaults",
+			providedStr: `{
+				"test": {
+					"txFee": 1000,
+					"minFee": 1000
+				}
+			}`,
+			defaultConfig: testConfig{TxFee: 100, MinFee: 100},
+			wantConfig:    testConfig{TxFee: 1000, MinFee: 1000},
+		},
+		{
+			name: "override one default value",
+			providedStr: `{
+				"test": {
+					"txFee": 1000
+				}
+			}`,
+			defaultConfig: testConfig{TxFee: 100, MinFee: 100},
+			wantConfig:    testConfig{TxFee: 1000, MinFee: 100},
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			r := require.New(t)
+			c, err := NewConfig([]byte(test.providedStr))
+			r.NoError(err)
+			testConfig, err := GetConfig(c, "test", test.defaultConfig)
+			r.NoError(err)
+			r.Equal(test.wantConfig, testConfig)
+		})
+	}
+}