Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

all: remove uses of untyped golang-lru #765

Merged
merged 1 commit into from
Dec 21, 2024
Merged

all: remove uses of untyped golang-lru #765

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 10 additions & 22 deletions XDCx/XDCx.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ import (
"github.com/XinFinOrg/XDPoSChain/XDCx/tradingstate"
"github.com/XinFinOrg/XDPoSChain/XDCxDAO"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/lru"
"github.com/XinFinOrg/XDPoSChain/common/prque"
"github.com/XinFinOrg/XDPoSChain/consensus"
"github.com/XinFinOrg/XDPoSChain/core/state"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/p2p"
"github.com/XinFinOrg/XDPoSChain/rpc"
lru "github.com/hashicorp/golang-lru"
"golang.org/x/sync/syncmap"
)

Expand Down Expand Up @@ -59,8 +59,8 @@ type XDCX struct {

sdkNode bool
settings syncmap.Map // holds configuration settings that can be dynamically changed
tokenDecimalCache *lru.Cache
orderCache *lru.Cache
tokenDecimalCache *lru.Cache[common.Address, *big.Int]
orderCache *lru.Cache[common.Hash, map[common.Hash]tradingstate.OrderHistoryItem]
}

func (XDCx *XDCX) Protocols() []p2p.Protocol {
Expand Down Expand Up @@ -94,19 +94,11 @@ func NewMongoDBEngine(cfg *Config) *XDCxDAO.MongoDatabase {
}

func New(cfg *Config) *XDCX {
tokenDecimalCache, err := lru.New(defaultCacheLimit)
if err != nil {
log.Warn("[XDCx-New] fail to create new lru for token decimal", "error", err)
}
orderCache, err := lru.New(tradingstate.OrderCacheLimit)
if err != nil {
log.Warn("[XDCx-New] fail to create new lru for order", "error", err)
}
XDCX := &XDCX{
orderNonce: make(map[common.Address]*big.Int),
Triegc: prque.New(nil),
tokenDecimalCache: tokenDecimalCache,
orderCache: orderCache,
tokenDecimalCache: lru.NewCache[common.Address, *big.Int](defaultCacheLimit),
orderCache: lru.NewCache[common.Hash, map[common.Hash]tradingstate.OrderHistoryItem](tradingstate.OrderCacheLimit),
}

// default DBEngine: levelDB
Expand Down Expand Up @@ -607,12 +599,9 @@ func (XDCx *XDCX) GetTradingStateRoot(block *types.Block, author common.Address)
}

func (XDCx *XDCX) UpdateOrderCache(baseToken, quoteToken common.Address, orderHash common.Hash, txhash common.Hash, lastState tradingstate.OrderHistoryItem) {
var orderCacheAtTxHash map[common.Hash]tradingstate.OrderHistoryItem
c, ok := XDCx.orderCache.Get(txhash)
if !ok || c == nil {
orderCacheAtTxHash, ok := XDCx.orderCache.Get(txhash)
if !ok || orderCacheAtTxHash == nil {
orderCacheAtTxHash = make(map[common.Hash]tradingstate.OrderHistoryItem)
} else {
orderCacheAtTxHash = c.(map[common.Hash]tradingstate.OrderHistoryItem)
}
orderKey := tradingstate.GetOrderHistoryKey(baseToken, quoteToken, orderHash)
_, ok = orderCacheAtTxHash[orderKey]
Expand All @@ -629,16 +618,15 @@ func (XDCx *XDCX) RollbackReorgTxMatch(txhash common.Hash) error {
items := db.GetListItemByTxHash(txhash, &tradingstate.OrderItem{})
if items != nil {
for _, order := range items.([]*tradingstate.OrderItem) {
c, ok := XDCx.orderCache.Get(txhash)
log.Debug("XDCx reorg: rollback order", "txhash", txhash.Hex(), "order", tradingstate.ToJSON(order), "orderHistoryItem", c)
if !ok {
orderCacheAtTxHash, ok := XDCx.orderCache.Get(txhash)
log.Debug("XDCx reorg: rollback order", "txhash", txhash.Hex(), "order", tradingstate.ToJSON(order), "orderHistoryItem", orderCacheAtTxHash)
if !ok || orderCacheAtTxHash == nil {
log.Debug("XDCx reorg: remove order due to no orderCache", "order", tradingstate.ToJSON(order))
if err := db.DeleteObject(order.Hash, &tradingstate.OrderItem{}); err != nil {
log.Crit("SDKNode: failed to remove reorg order", "err", err.Error(), "order", tradingstate.ToJSON(order))
}
continue
}
orderCacheAtTxHash := c.(map[common.Hash]tradingstate.OrderHistoryItem)
orderHistoryItem := orderCacheAtTxHash[tradingstate.GetOrderHistoryKey(order.BaseToken, order.QuoteToken, order.Hash)]
if (orderHistoryItem == tradingstate.OrderHistoryItem{}) {
log.Debug("XDCx reorg: remove order due to empty orderHistory", "order", tradingstate.ToJSON(order))
Expand Down
11 changes: 5 additions & 6 deletions XDCx/token.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,14 @@ import (
"math/big"
"strings"

"github.com/XinFinOrg/XDPoSChain/contracts/XDCx/contract"
"github.com/XinFinOrg/XDPoSChain/core"
"github.com/XinFinOrg/XDPoSChain/log"

"github.com/XinFinOrg/XDPoSChain"
"github.com/XinFinOrg/XDPoSChain/accounts/abi"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/consensus"
"github.com/XinFinOrg/XDPoSChain/contracts/XDCx/contract"
"github.com/XinFinOrg/XDPoSChain/core"
"github.com/XinFinOrg/XDPoSChain/core/state"
"github.com/XinFinOrg/XDPoSChain/log"
)

// GetTokenAbi return token abi
Expand Down Expand Up @@ -45,8 +44,8 @@ func RunContract(chain consensus.ChainContext, statedb *state.StateDB, contractA
}

func (XDCx *XDCX) GetTokenDecimal(chain consensus.ChainContext, statedb *state.StateDB, tokenAddr common.Address) (*big.Int, error) {
if tokenDecimal, ok := XDCx.tokenDecimalCache.Get(tokenAddr); ok {
return tokenDecimal.(*big.Int), nil
if tokenDecimal, ok := XDCx.tokenDecimalCache.Get(tokenAddr); ok && tokenDecimal != nil {
return tokenDecimal, nil
}
if tokenAddr == common.XDCNativeAddressBinary {
XDCx.tokenDecimalCache.Add(tokenAddr, common.BasePrice)
Expand Down
15 changes: 4 additions & 11 deletions XDCx/tradingstate/database.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/trie"
lru "github.com/hashicorp/golang-lru"
)

// Trie cache generation limit after which to evic trie nodes from memory.
Expand All @@ -33,9 +32,6 @@ const (
// Number of past tries to keep. This value is chosen such that
// reasonable chain reorg depths will hit an existing trie.
maxPastTries = 12

// Number of codehash->size associations to keep.
codeSizeCacheSize = 100000
)

// Database wraps access to tries and contract code.
Expand Down Expand Up @@ -79,18 +75,15 @@ type Trie interface {
// intermediate trie-node memory pool between the low level storage layer and the
// high level trie abstraction.
func NewDatabase(db ethdb.Database) Database {
csc, _ := lru.New(codeSizeCacheSize)
return &cachingDB{
db: trie.NewDatabase(db),
codeSizeCache: csc,
db: trie.NewDatabase(db),
}
}

type cachingDB struct {
db *trie.Database
mu sync.Mutex
pastTries []*XDCXTrie
codeSizeCache *lru.Cache
db *trie.Database
mu sync.Mutex
pastTries []*XDCXTrie
}

// OpenTrie opens the main account trie.
Expand Down
6 changes: 0 additions & 6 deletions XDCxDAO/leveldb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,8 @@ import (

"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"

"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
lru "github.com/hashicorp/golang-lru"
)

type BatchItem struct {
Expand All @@ -21,7 +19,6 @@ type BatchItem struct {
type BatchDatabase struct {
db ethdb.Database
emptyKey []byte
cacheItems *lru.Cache // Cache for reading
lock sync.RWMutex
cacheLimit int
Debug bool
Expand All @@ -44,11 +41,8 @@ func NewBatchDatabaseWithEncode(datadir string, cacheLimit int) *BatchDatabase {
itemCacheLimit = cacheLimit
}

cacheItems, _ := lru.New(itemCacheLimit)

batchDB := &BatchDatabase{
db: db,
cacheItems: cacheItems,
emptyKey: EmptyKey(), // pre alloc for comparison
cacheLimit: itemCacheLimit,
}
Expand Down
7 changes: 3 additions & 4 deletions XDCxDAO/mongodb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ import (
"github.com/XinFinOrg/XDPoSChain/XDCx/tradingstate"
"github.com/XinFinOrg/XDPoSChain/XDCxlending/lendingstate"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/lru"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
lru "github.com/hashicorp/golang-lru"
)

const (
Expand All @@ -32,7 +32,7 @@ type MongoDatabase struct {
Session *mgo.Session
dbName string
emptyKey []byte
cacheItems *lru.Cache // Cache for reading
cacheItems *lru.Cache[string, interface{}] // Cache for reading
orderBulk *mgo.Bulk
tradeBulk *mgo.Bulk
epochPriceBulk *mgo.Bulk
Expand Down Expand Up @@ -64,12 +64,11 @@ func NewMongoDatabase(session *mgo.Session, dbName string, mongoURL string, repl
if cacheLimit > 0 {
itemCacheLimit = cacheLimit
}
cacheItems, _ := lru.New(itemCacheLimit)

db := &MongoDatabase{
Session: session,
dbName: dbName,
cacheItems: cacheItems,
cacheItems: lru.NewCache[string, interface{}](itemCacheLimit),
}
if err := db.EnsureIndexes(); err != nil {
return nil, err
Expand Down
39 changes: 15 additions & 24 deletions XDCxlending/XDCxlending.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ import (
"github.com/XinFinOrg/XDPoSChain/XDCxDAO"
"github.com/XinFinOrg/XDPoSChain/XDCxlending/lendingstate"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/lru"
"github.com/XinFinOrg/XDPoSChain/common/prque"
"github.com/XinFinOrg/XDPoSChain/consensus"
"github.com/XinFinOrg/XDPoSChain/core/state"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/p2p"
"github.com/XinFinOrg/XDPoSChain/rpc"
lru "github.com/hashicorp/golang-lru"
)

const (
Expand All @@ -42,8 +42,8 @@ type Lending struct {
orderNonce map[common.Address]*big.Int

XDCx *XDCx.XDCX
lendingItemHistory *lru.Cache
lendingTradeHistory *lru.Cache
lendingItemHistory *lru.Cache[common.Hash, map[common.Hash]lendingstate.LendingItemHistoryItem]
lendingTradeHistory *lru.Cache[common.Hash, map[common.Hash]lendingstate.LendingTradeHistoryItem]
}

func (l *Lending) Protocols() []p2p.Protocol {
Expand All @@ -62,13 +62,11 @@ func (l *Lending) Stop() error {
}

func New(XDCx *XDCx.XDCX) *Lending {
itemCache, _ := lru.New(defaultCacheLimit)
lendingTradeCache, _ := lru.New(defaultCacheLimit)
lending := &Lending{
orderNonce: make(map[common.Address]*big.Int),
Triegc: prque.New(nil),
lendingItemHistory: itemCache,
lendingTradeHistory: lendingTradeCache,
lendingItemHistory: lru.NewCache[common.Hash, map[common.Hash]lendingstate.LendingItemHistoryItem](defaultCacheLimit),
lendingTradeHistory: lru.NewCache[common.Hash, map[common.Hash]lendingstate.LendingTradeHistoryItem](defaultCacheLimit),
}
lending.StateCache = lendingstate.NewDatabase(XDCx.GetLevelDB())
lending.XDCx = XDCx
Expand Down Expand Up @@ -705,12 +703,9 @@ func (l *Lending) GetLendingStateRoot(block *types.Block, author common.Address)
}

func (l *Lending) UpdateLendingItemCache(LendingToken, CollateralToken common.Address, hash common.Hash, txhash common.Hash, lastState lendingstate.LendingItemHistoryItem) {
var lendingCacheAtTxHash map[common.Hash]lendingstate.LendingItemHistoryItem
c, ok := l.lendingItemHistory.Get(txhash)
if !ok || c == nil {
lendingCacheAtTxHash, ok := l.lendingItemHistory.Get(txhash)
if !ok || lendingCacheAtTxHash == nil {
lendingCacheAtTxHash = make(map[common.Hash]lendingstate.LendingItemHistoryItem)
} else {
lendingCacheAtTxHash = c.(map[common.Hash]lendingstate.LendingItemHistoryItem)
}
orderKey := lendingstate.GetLendingItemHistoryKey(LendingToken, CollateralToken, hash)
_, ok = lendingCacheAtTxHash[orderKey]
Expand All @@ -722,11 +717,9 @@ func (l *Lending) UpdateLendingItemCache(LendingToken, CollateralToken common.Ad

func (l *Lending) UpdateLendingTradeCache(hash common.Hash, txhash common.Hash, lastState lendingstate.LendingTradeHistoryItem) {
var lendingCacheAtTxHash map[common.Hash]lendingstate.LendingTradeHistoryItem
c, ok := l.lendingTradeHistory.Get(txhash)
if !ok || c == nil {
lendingCacheAtTxHash, ok := l.lendingTradeHistory.Get(txhash)
if !ok || lendingCacheAtTxHash == nil {
lendingCacheAtTxHash = make(map[common.Hash]lendingstate.LendingTradeHistoryItem)
} else {
lendingCacheAtTxHash = c.(map[common.Hash]lendingstate.LendingTradeHistoryItem)
}
_, ok = lendingCacheAtTxHash[hash]
if !ok {
Expand All @@ -743,16 +736,15 @@ func (l *Lending) RollbackLendingData(txhash common.Hash) error {
items := db.GetListItemByTxHash(txhash, &lendingstate.LendingItem{})
if items != nil {
for _, item := range items.([]*lendingstate.LendingItem) {
c, ok := l.lendingItemHistory.Get(txhash)
log.Debug("XDCxlending reorg: rollback lendingItem", "txhash", txhash.Hex(), "item", lendingstate.ToJSON(item), "lendingItemHistory", c)
if !ok {
cacheAtTxHash, ok := l.lendingItemHistory.Get(txhash)
log.Debug("XDCxlending reorg: rollback lendingItem", "txhash", txhash.Hex(), "item", lendingstate.ToJSON(item), "lendingItemHistory", cacheAtTxHash)
if !ok || cacheAtTxHash == nil {
log.Debug("XDCxlending reorg: remove item due to no lendingItemHistory", "item", lendingstate.ToJSON(item))
if err := db.DeleteObject(item.Hash, &lendingstate.LendingItem{}); err != nil {
return fmt.Errorf("failed to remove reorg LendingItem. Err: %v . Item: %s", err.Error(), lendingstate.ToJSON(item))
}
continue
}
cacheAtTxHash := c.(map[common.Hash]lendingstate.LendingItemHistoryItem)
lendingItemHistory := cacheAtTxHash[lendingstate.GetLendingItemHistoryKey(item.LendingToken, item.CollateralToken, item.Hash)]
if (lendingItemHistory == lendingstate.LendingItemHistoryItem{}) {
log.Debug("XDCxlending reorg: remove item due to empty lendingItemHistory", "item", lendingstate.ToJSON(item))
Expand All @@ -776,16 +768,15 @@ func (l *Lending) RollbackLendingData(txhash common.Hash) error {
items = db.GetListItemByTxHash(txhash, &lendingstate.LendingTrade{})
if items != nil {
for _, trade := range items.([]*lendingstate.LendingTrade) {
c, ok := l.lendingTradeHistory.Get(txhash)
log.Debug("XDCxlending reorg: rollback LendingTrade", "txhash", txhash.Hex(), "trade", lendingstate.ToJSON(trade), "LendingTradeHistory", c)
if !ok {
cacheAtTxHash, ok := l.lendingTradeHistory.Get(txhash)
log.Debug("XDCxlending reorg: rollback LendingTrade", "txhash", txhash.Hex(), "trade", lendingstate.ToJSON(trade), "LendingTradeHistory", cacheAtTxHash)
if !ok || cacheAtTxHash == nil {
log.Debug("XDCxlending reorg: remove trade due to no LendingTradeHistory", "trade", lendingstate.ToJSON(trade))
if err := db.DeleteObject(trade.Hash, &lendingstate.LendingTrade{}); err != nil {
return fmt.Errorf("failed to remove reorg LendingTrade. Err: %v . Trade: %s", err.Error(), lendingstate.ToJSON(trade))
}
continue
}
cacheAtTxHash := c.(map[common.Hash]lendingstate.LendingTradeHistoryItem)
lendingTradeHistoryItem := cacheAtTxHash[trade.Hash]
if (lendingTradeHistoryItem == lendingstate.LendingTradeHistoryItem{}) {
log.Debug("XDCxlending reorg: remove trade due to empty LendingTradeHistory", "trade", lendingstate.ToJSON(trade))
Expand Down
16 changes: 5 additions & 11 deletions XDCxlending/lendingstate/database.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/trie"
lru "github.com/hashicorp/golang-lru"
)

// Trie cache generation limit after which to evic trie nodes from memory.
Expand All @@ -33,9 +32,6 @@ const (
// Number of past tries to keep. This value is chosen such that
// reasonable chain reorg depths will hit an existing trie.
maxPastTries = 12

// Number of codehash->size associations to keep.
codeSizeCacheSize = 100000
)

// Database wraps access to tries and contract code.
Expand Down Expand Up @@ -78,18 +74,16 @@ type Trie interface {
// intermediate trie-node memory pool between the low level storage layer and the
// high level trie abstraction.
func NewDatabase(db ethdb.Database) Database {
csc, _ := lru.New(codeSizeCacheSize)
return &cachingDB{
db: trie.NewDatabase(db),
codeSizeCache: csc,
db: trie.NewDatabase(db),
// codeSizeCache: csc,
}
}

type cachingDB struct {
db *trie.Database
mu sync.Mutex
pastTries []*XDCXTrie
codeSizeCache *lru.Cache
db *trie.Database
mu sync.Mutex
pastTries []*XDCXTrie
}

// OpenTrie opens the main account trie.
Expand Down
Loading
Loading