Skip to content

Commit

Permalink
Implement Configurable Logging (#50)
Browse files Browse the repository at this point in the history 
* Configurable logger

* Configurable logger

* Minor fixes

* Fix linter

* Add log to GetActorCode

* Delete unnecessary getSafeLogger
  • Loading branch information
@lucaslopezf
lucaslopezf authored Sep 27, 2023
1 parent 0c29453 commit 7788d01
Show file tree
Hide file tree
Showing 16 changed files with 152 additions and 109 deletions.
7 changes: 5 additions & 2 deletions actors/actor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"github.com/filecoin-project/go-state-types/manifest"
filTypes "github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
logger2 "github.com/zondax/fil-parser/logger"
"github.com/zondax/fil-parser/parser"
"github.com/zondax/fil-parser/parser/helper"
"github.com/zondax/fil-parser/types"
Expand All @@ -12,11 +13,13 @@ import (

type ActorParser struct {
helper *helper.Helper
logger *zap.Logger
}

func NewActorParser(helper *helper.Helper) *ActorParser {
func NewActorParser(helper *helper.Helper, logger *zap.Logger) *ActorParser {
return &ActorParser{
helper: helper,
logger: logger2.GetSafeLogger(logger),
}
}

Expand All @@ -34,7 +37,7 @@ func (p *ActorParser) GetMetadata(txType string, msg *parser.LotusMessage, mainM

c, err := cid.Parse(actorCode)
if err != nil {
zap.S().Errorf("Could not parse actor code: %v", err)
p.logger.Sugar().Errorf("Could not parse actor code: %v", err)
return metadata, nil, err
}

Expand Down
35 changes: 19 additions & 16 deletions actors/cache/actors_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
cmap "github.com/orcaman/concurrent-map"
"github.com/zondax/fil-parser/actors/cache/impl"
"github.com/zondax/fil-parser/actors/cache/impl/common"
logger2 "github.com/zondax/fil-parser/logger"
"github.com/zondax/fil-parser/types"
"go.uber.org/zap"
"strings"
Expand All @@ -25,37 +26,39 @@ var SystemActorsId = map[string]bool{
"f099": true,
}

func SetupActorsCache(dataSource common.DataSource) (*ActorsCache, error) {
func SetupActorsCache(dataSource common.DataSource, logger *zap.Logger) (*ActorsCache, error) {
var offlineCache IActorsCache
var onChainCache impl.OnChain

err := onChainCache.NewImpl(dataSource)
logger = logger2.GetSafeLogger(logger)
err := onChainCache.NewImpl(dataSource, logger)
if err != nil {
return nil, err
}

// Try kvStore cache, if it fails, on-memory cache
var kvStoreCache impl.KVStore
err = kvStoreCache.NewImpl(dataSource)
err = kvStoreCache.NewImpl(dataSource, logger)
if err == nil {
offlineCache = &kvStoreCache
} else {
zap.S().Warn("[ActorsCache] - Unable to initialize kv store cache. Using on-memory cache")
logger.Sugar().Warn("[ActorsCache] - Unable to initialize kv store cache. Using on-memory cache")
var inMemoryCache impl.Memory
err = inMemoryCache.NewImpl(dataSource)
err = inMemoryCache.NewImpl(dataSource, logger)
if err != nil {
zap.S().Errorf("[ActorsCache] - Unable to initialize on-memory cache: %s", err.Error())
logger.Sugar().Errorf("[ActorsCache] - Unable to initialize on-memory cache: %s", err.Error())
return nil, err
}
offlineCache = &inMemoryCache
}

zap.S().Infof("[ActorsCache] - Actors cache initialized. Offline cache implementation: %s", offlineCache.ImplementationType())
logger.Sugar().Infof("[ActorsCache] - Actors cache initialized. Offline cache implementation: %s", offlineCache.ImplementationType())

return &ActorsCache{
offlineCache: offlineCache,
onChainCache: &onChainCache,
badAddress: cmap.New(),
logger: logger,
}, nil
}

Expand All @@ -75,11 +78,11 @@ func (a *ActorsCache) GetActorCode(add address.Address, key filTypes.TipSetKey)
return actorCode, nil
}

zap.S().Debugf("[ActorsCache] - Unable to retrieve actor code from offline cache for address %s. Trying on-chain cache", add.String())
a.logger.Sugar().Debugf("[ActorsCache] - Unable to retrieve actor code from offline cache for address %s. Trying on-chain cache", add.String())
// Try on-chain cache
actorCode, err = a.onChainCache.GetActorCode(add, key)
if err != nil {
zap.S().Error("[ActorsCache] - Unable to retrieve actor code from node: %s", err.Error())
a.logger.Sugar().Error("[ActorsCache] - Unable to retrieve actor code from node: %s", err.Error())
if strings.Contains(err.Error(), "actor not found") {
a.badAddress.Set(add.String(), true)
}
Expand All @@ -93,7 +96,7 @@ func (a *ActorsCache) GetActorCode(add address.Address, key filTypes.TipSetKey)
})

if err != nil {
zap.S().Errorf("[ActorsCache] - Unable to store address info: %s", err.Error())
a.logger.Sugar().Errorf("[ActorsCache] - Unable to store address info: %s", err.Error())
return "", err
}

Expand All @@ -116,12 +119,12 @@ func (a *ActorsCache) GetRobustAddress(add address.Address) (string, error) {
return "", fmt.Errorf("address %s is flagged as bad", add.String())
}

zap.S().Debugf("[ActorsCache] - Unable to retrieve robust address from offline cache for address %s. Trying on-chain cache", add.String())
a.logger.Sugar().Debugf("[ActorsCache] - Unable to retrieve robust address from offline cache for address %s. Trying on-chain cache", add.String())

// Try on-chain cache
robust, err = a.onChainCache.GetRobustAddress(add)
if err != nil {
zap.S().Errorf("[ActorsCache] - Unable to retrieve actor code from node: %s", err.Error())
a.logger.Sugar().Errorf("[ActorsCache] - Unable to retrieve actor code from node: %s", err.Error())
return "", err
}

Expand All @@ -131,7 +134,7 @@ func (a *ActorsCache) GetRobustAddress(add address.Address) (string, error) {
})

if err != nil {
zap.S().Errorf("[ActorsCache] - Unable to store address info: %s", err.Error())
a.logger.Sugar().Errorf("[ActorsCache] - Unable to store address info: %s", err.Error())
return "", err
}

Expand All @@ -150,12 +153,12 @@ func (a *ActorsCache) GetShortAddress(add address.Address) (string, error) {
return "", fmt.Errorf("address %s is flagged as bad", add.String())
}

zap.S().Debugf("[ActorsCache] - Unable to retrieve short address from offline cache for address %s. Trying on-chain cache", add.String())
a.logger.Sugar().Debugf("[ActorsCache] - Unable to retrieve short address from offline cache for address %s. Trying on-chain cache", add.String())

// Try on-chain cache
short, err = a.onChainCache.GetShortAddress(add)
if err != nil {
zap.S().Error("[ActorsCache] - Unable to retrieve actor code from node: %s", err.Error())
a.logger.Sugar().Error("[ActorsCache] - Unable to retrieve actor code from node: %s", err.Error())
return "", err
}

Expand All @@ -165,7 +168,7 @@ func (a *ActorsCache) GetShortAddress(add address.Address) (string, error) {
})

if err != nil {
zap.S().Errorf("[ActorsCache] - Unable to store address info: %s", err.Error())
a.logger.Sugar().Errorf("[ActorsCache] - Unable to store address info: %s", err.Error())
return "", err
}

Expand Down
48 changes: 26 additions & 22 deletions actors/cache/impl/kv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
nats2 "github.com/nats-io/nats.go"
"github.com/zondax/fil-parser/actors/cache/impl/common"
logger2 "github.com/zondax/fil-parser/logger"
"go.uber.org/zap"
"strconv"

Expand Down Expand Up @@ -31,16 +32,19 @@ type KVStore struct {
db *gorm.DB
nats *znats.ComponentNats
Config common.DataSourceConfig
logger *zap.Logger
}

func (m *KVStore) ImplementationType() string {
return KvStoreImpl
}

func (m *KVStore) NewImpl(source common.DataSource) error {
func (m *KVStore) NewImpl(source common.DataSource, logger *zap.Logger) error {
m.logger = logger2.GetSafeLogger(logger)

// Database is mandatory
if source.Db == nil {
zap.S().Warn("[ActorsCache] - Database ptr is nil. Database cache is disabled")
m.logger.Sugar().Warn("[ActorsCache] - Database ptr is nil. Database cache is disabled")
return fmt.Errorf("database ptr is nil")
}

Expand All @@ -49,13 +53,13 @@ func (m *KVStore) NewImpl(source common.DataSource) error {

// Nats is mandatory
if source.Config.Nats == nil {
zap.S().Errorf("[ActorsCache] - Nats ptr is nil. Nats cache is disabled")
m.logger.Sugar().Errorf("[ActorsCache] - Nats ptr is nil. Nats cache is disabled")
return fmt.Errorf("nats ptr is nil")
}

nats, err := znats.NewNatsComponent(*source.Config.Nats)
if err != nil {
zap.S().Panicf("[ActorsCache] - Error creating nats component: %s", err.Error())
m.logger.Sugar().Panicf("[ActorsCache] - Error creating nats component: %s", err.Error())
}

m.nats = nats
Expand All @@ -74,7 +78,7 @@ func (m *KVStore) NewImpl(source common.DataSource) error {
})

if err != nil {
zap.S().Errorf("[ActorsCache] - Error creating short robust kv store: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error creating short robust kv store: %s", err.Error())
return err
}

Expand All @@ -87,7 +91,7 @@ func (m *KVStore) NewImpl(source common.DataSource) error {
})

if err != nil {
zap.S().Errorf("[ActorsCache] - Error creating robust short kv store: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error creating robust short kv store: %s", err.Error())
return err
}

Expand All @@ -100,7 +104,7 @@ func (m *KVStore) NewImpl(source common.DataSource) error {
})

if err != nil {
zap.S().Errorf("[ActorsCache] - Error creating short actor type kv store: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error creating short actor type kv store: %s", err.Error())
return err
}

Expand All @@ -113,19 +117,19 @@ func (m *KVStore) NewImpl(source common.DataSource) error {
})

if err != nil {
zap.S().Errorf("[ActorsCache] - Error creating state kv store: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error creating state kv store: %s", err.Error())
return err
}

_, err = m.nats.MapKVStore[StateStoreName].Store.Put(backFillInProgressKey, []byte("false"))
if err != nil {
zap.S().Warnf("[ActorsCache] - Error setting backfill in progress key: %s", err)
m.logger.Sugar().Warnf("[ActorsCache] - Error setting backfill in progress key: %s", err)
}

if m.isNatsCacheEmpty() {
err = m.BackFill()
if err != nil {
zap.S().Errorf("[ActorsCache] - Error backfilling cache: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error backfilling cache: %s", err.Error())
}
}

Expand All @@ -144,7 +148,7 @@ func (m *KVStore) isNatsCacheEmpty() bool {
for _, store := range stores {
keys, err := m.nats.MapKVStore[store].Store.Keys()
if err != nil {
zap.S().Errorf("[ActorsCache] - Error getting keys from store %s: %s", store, err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error getting keys from store %s: %s", store, err.Error())
continue
}

Expand All @@ -169,32 +173,32 @@ func (m *KVStore) BackFill() error {
// Check if backfill is already in progress
backfillInProgress, err := m.nats.MapKVStore[StateStoreName].Store.Get(backFillInProgressKey)
if err != nil {
zap.S().Errorf("[ActorsCache] - Error getting backfill in progress key: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error getting backfill in progress key: %s", err.Error())
return err
}

inProgress, err := strconv.ParseBool(string(backfillInProgress.Value()))
if err != nil {
zap.S().Errorf("[ActorsCache] - Error parsing backfill in progress key: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error parsing backfill in progress key: %s", err.Error())
return err
}

if inProgress {
zap.S().Info("[ActorsCache] - Backfill already in progress")
m.logger.Sugar().Info("[ActorsCache] - Backfill already in progress")
return nil
}

// Set backfill in progress
_, err = m.nats.MapKVStore[StateStoreName].Store.Put(backFillInProgressKey, []byte("true"))
if err != nil {
zap.S().Errorf("[ActorsCache] - Error setting backfill in progress key: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error setting backfill in progress key: %s", err.Error())
return err
}

// Copy the content of the database into the kv store
addresses := make([]types.AddressInfo, 0)
m.db.Table(m.Config.InputTableName).Find(&addresses)
zap.S().Infof("[ActorsCache] - Backfilling %d addresses", len(addresses))
m.logger.Sugar().Infof("[ActorsCache] - Backfilling %d addresses", len(addresses))

for _, add := range addresses {
m.storeShortRobust(add.Short, add.Robust)
Expand All @@ -205,18 +209,18 @@ func (m *KVStore) BackFill() error {
// Set backfill finished
_, err = m.nats.MapKVStore[StateStoreName].Store.Put(backFillInProgressKey, []byte("false"))
if err != nil {
zap.S().Errorf("[ActorsCache] - Error setting backfill in progress key: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error setting backfill in progress key: %s", err.Error())
return err
}

zap.S().Info("[ActorsCache] - Backfill finished")
m.logger.Sugar().Info("[ActorsCache] - Backfill finished")
return nil
}

func (m *KVStore) GetActorCode(address address.Address, key filTypes.TipSetKey) (string, error) {
shortAddress, err := m.GetShortAddress(address)
if err != nil {
fmt.Printf("[ActorsCache] - Error getting short address: %s\n", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error getting short address: %s\n", err.Error())
return cid.Undef.String(), err
}

Expand Down Expand Up @@ -285,7 +289,7 @@ func (m *KVStore) storeRobustShort(robust string, short string) {

_, err := m.nats.MapKVStore[RobustShortStoreName].Store.Put(robust, []byte(short))
if err != nil {
zap.S().Errorf("[ActorsCache] - Error storing robust short in kv store: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error storing robust short in kv store: %s", err.Error())
}
}

Expand All @@ -296,7 +300,7 @@ func (m *KVStore) storeShortRobust(short string, robust string) {

_, err := m.nats.MapKVStore[ShortRobustStoreName].Store.Put(short, []byte(robust))
if err != nil {
zap.S().Errorf("[ActorsCache] - Error storing short robust in kv store: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error storing short robust in kv store: %s", err.Error())
}

}
Expand All @@ -314,6 +318,6 @@ func (m *KVStore) storeActorCode(shortAddress string, cid string) {

_, err := m.nats.MapKVStore[ShortCidStoreName].Store.Put(shortAddress, []byte(cid))
if err != nil {
zap.S().Errorf("[ActorsCache] - Error storing actor code in kv store: %s", err.Error())
m.logger.Sugar().Errorf("[ActorsCache] - Error storing actor code in kv store: %s", err.Error())
}
}
3 changes: 2 additions & 1 deletion actors/cache/impl/memory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
cmap "github.com/orcaman/concurrent-map"
"github.com/zondax/fil-parser/actors/cache/impl/common"
"github.com/zondax/fil-parser/types"
"go.uber.org/zap"
)

const InMemoryImpl = "in-memory"
Expand All @@ -19,7 +20,7 @@ type Memory struct {
shortRobustMap cmap.ConcurrentMap
}

func (m *Memory) NewImpl(source common.DataSource) error {
func (m *Memory) NewImpl(source common.DataSource, _ *zap.Logger) error {
m.shortCidMap = cmap.New()
m.robustShortMap = cmap.New()
m.shortRobustMap = cmap.New()
Expand Down
Loading

0 comments on commit 7788d01

Please sign in to comment.