Merge pull request #201 from ipfs/feat/log-memory-blockstore-consumption

Log unverified blockstore memory consumption

requestmanager/asyncloader/unverifiedblockstore/unverifiedblockstore.go

@@ -2,10 +2,12 @@ package unverifiedblockstore
 
 import (
  "fmt"
-
+ logging "github.com/ipfs/go-log/v2"
  ipld "github.com/ipld/go-ipld-prime"
 )
 
+var log = logging.Logger("gs-unverifiedbs")
+
 type settableWriter interface {
  SetBytes([]byte) error
 }
@@ -15,6 +17,7 @@ type settableWriter interface {
 type UnverifiedBlockStore struct {
  inMemoryBlocks map[ipld.Link][]byte
  storer ipld.Storer
+ dataSize uint64
 }
 
 // New initializes a new unverified store with the given storer function for writing
@@ -30,21 +33,26 @@ func New(storer ipld.Storer) *UnverifiedBlockStore {
 // comes in as part of a traversal.
 func (ubs *UnverifiedBlockStore) AddUnverifiedBlock(lnk ipld.Link, data []byte) {
  ubs.inMemoryBlocks[lnk] = data
+ ubs.dataSize = ubs.dataSize + uint64(len(data))
+ log.Debugw("added in-memory block", "total_queued_bytes", ubs.dataSize)
 }
 
 // PruneBlocks removes blocks from the unverified store without committing them,
 // if the passed in function returns true for the given link
 func (ubs *UnverifiedBlockStore) PruneBlocks(shouldPrune func(ipld.Link) bool) {
- for link := range ubs.inMemoryBlocks {
+ for link,data := range ubs.inMemoryBlocks {
  if shouldPrune(link) {
  delete(ubs.inMemoryBlocks, link)
+ ubs.dataSize = ubs.dataSize - uint64(len(data))
  }
  }
+ log.Debugw("finished pruning in-memory blocks", "total_queued_bytes", ubs.dataSize)
 }
 
 // PruneBlock deletes an individual block from the store
 func (ubs *UnverifiedBlockStore) PruneBlock(link ipld.Link) {
  delete(ubs.inMemoryBlocks, link)
+ ubs.dataSize = ubs.dataSize - uint64(len(ubs.inMemoryBlocks[link]))
 }
 
 // VerifyBlock verifies the data for the given link as being part of a traversal,
@@ -55,6 +63,8 @@ func (ubs *UnverifiedBlockStore) VerifyBlock(lnk ipld.Link) ([]byte, error) {
  return nil, fmt.Errorf("Block not found")
  }
  delete(ubs.inMemoryBlocks, lnk)
+ ubs.dataSize = ubs.dataSize - uint64(len(data))
+
  buffer, committer, err := ubs.storer(ipld.LinkContext{})
  if err != nil {
  return nil, err