Merge commit '87ecb92fceb58497268f7460ac1c877a97b056df' into blockser…

…vice-fix

Conflicts:
	blocks/blockstore/blockstore.go
	blockservice/blockservice.go

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>

ISSUE_TEMPLATE.md

@@ -1,18 +1,23 @@
-Version/Platform/Processor information (from `ipfs version --all`):
+<!-- Output From `ipfs version --all`) -->
+#### Version information:
 
+<!-- Bug, Feature, Enhancement, Etc -->
+#### Type:
 
+<!-- from `P0` "Critical" to `P5` "Relatively Unimportant") -->
+#### Priority:
 
-Type (bug, feature, meta, test failure, question):
-Area (api, commands, daemon, fuse, etc):
-Priority (from P0: operations on fire, to P4: functioning):
+#### Description:
 
-Description:
 
 
 
 
----------------------------------------------------
+
+
+<!--
 This is for you! Please read, and then delete this text before posting it.
 The go-ipfs issues are only for bug reports and directly actionable features.
-Check https://github.com/ipfs/community/blob/master/contributing.md#reporting-issues if that doesn't fit.
-Check https://github.com/ipfs/go-ipfs/blob/master/docs/github-issue-guide.md if you are not sure how to fill this issue.
+Read https://github.com/ipfs/community/blob/master/contributing.md#reporting-issues if your issue doesn't fit either of those categories.
+Read https://github.com/ipfs/go-ipfs/blob/master/docs/github-issue-guide.md if you are not sure how to fill in this issue.
+-->

README.md

@@ -106,8 +106,8 @@ If your operating system isn't officially supported, but you still want to try
 building ipfs anyways (it should work fine in most cases), you can do the
 following:
 
-- Install gx: `go get github.com/whyrusleeping/gx`
-- Install gx-go: `go get github.com/whyrusleeping/gx-go`
+- Install gx: `go get -u github.com/whyrusleeping/gx`
+- Install gx-go: `go get -u github.com/whyrusleeping/gx-go`
 - Fetch ipfs source: `go get -d github.com/ipfs/go-ipfs 2> /dev/null`
 - Enter source directory: `cd $GOPATH/src/github.com/ipfs/go-ipfs`
 - Install deps: `gx install`

assets/assets.go

@@ -8,10 +8,10 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/ipfs/go-ipfs/blocks/key"
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/core/coreunix"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
+	cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
 )
 
 // initDocPaths lists the paths for the docs we want to seed during --init
@@ -25,7 +25,7 @@ var initDocPaths = []string{
 }
 
 // SeedInitDocs adds the list of embedded init documentation to the passed node, pins it and returns the root key
-func SeedInitDocs(nd *core.IpfsNode) (*key.Key, error) {
+func SeedInitDocs(nd *core.IpfsNode) (*cid.Cid, error) {
 	return addAssetList(nd, initDocPaths)
 }
 
@@ -34,11 +34,11 @@ var initDirIndex = []string{
 	filepath.Join("..", "vendor", "dir-index-html-v1.0.0", "dir-index.html"),
 }
 
-func SeedInitDirIndex(nd *core.IpfsNode) (*key.Key, error) {
+func SeedInitDirIndex(nd *core.IpfsNode) (*cid.Cid, error) {
 	return addAssetList(nd, initDirIndex)
 }
 
-func addAssetList(nd *core.IpfsNode, l []string) (*key.Key, error) {
+func addAssetList(nd *core.IpfsNode, l []string) (*cid.Cid, error) {
 	dirb := uio.NewDirectory(nd.DAG)
 
 	for _, p := range l {
@@ -53,14 +53,18 @@ func addAssetList(nd *core.IpfsNode, l []string) (*key.Key, error) {
 		}
 
 		fname := filepath.Base(p)
-		k := key.B58KeyDecode(s)
-		if err := dirb.AddChild(nd.Context(), fname, k); err != nil {
+		c, err := cid.Decode(s)
+		if err != nil {
+			return nil, err
+		}
+
+		if err := dirb.AddChild(nd.Context(), fname, c); err != nil {
 			return nil, fmt.Errorf("assets: could not add '%s' as a child: %s", fname, err)
 		}
 	}
 
 	dir := dirb.GetNode()
-	dkey, err := nd.DAG.Add(dir)
+	dcid, err := nd.DAG.Add(dir)
 	if err != nil {
 		return nil, fmt.Errorf("assets: DAG.Add(dir) failed: %s", err)
 	}
@@ -73,5 +77,5 @@ func addAssetList(nd *core.IpfsNode, l []string) (*key.Key, error) {
 		return nil, fmt.Errorf("assets: Pinning flush failed: %s", err)
 	}
 
-	return &dkey, nil
+	return dcid, nil
 }

blocks/blocks.go

@@ -6,16 +6,18 @@ import (
 	"errors"
 	"fmt"
 
-	key "github.com/ipfs/go-ipfs/blocks/key"
+	key "gx/ipfs/Qmce4Y4zg3sYr7xKM5UueS67vhNni6EeWgCRnb7MbLJMew/go-key"
+
 	mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
+	cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
 )
 
 var ErrWrongHash = errors.New("data did not match given hash!")
 
 type Block interface {
 	Multihash() mh.Multihash
-	Data() []byte
+	RawData() []byte
 	Key() key.Key
 	String() string
 	Loggable() map[string]interface{}
@@ -49,10 +51,14 @@ func (b *BasicBlock) Multihash() mh.Multihash {
 	return b.multihash
 }
 
-func (b *BasicBlock) Data() []byte {
+func (b *BasicBlock) RawData() []byte {
 	return b.data
 }
 
+func (b *BasicBlock) Cid() *cid.Cid {
+	return cid.NewCidV0(b.multihash)
+}
+
 // Key returns the block's Multihash as a Key value.
 func (b *BasicBlock) Key() key.Key {
 	return key.Key(b.multihash)

blocks/blocks_test.go

@@ -25,7 +25,7 @@ func TestData(t *testing.T) {
 	data := []byte("some data")
 	block := NewBlock(data)
 
-	if !bytes.Equal(block.Data(), data) {
+	if !bytes.Equal(block.RawData(), data) {
 		t.Error("data is wrong")
 	}
 }

blocks/blockstore/arc_cache.go

@@ -1,26 +1,34 @@
 package blockstore
 
 import (
+	key "gx/ipfs/Qmce4Y4zg3sYr7xKM5UueS67vhNni6EeWgCRnb7MbLJMew/go-key"
+
 	"github.com/ipfs/go-ipfs/blocks"
-	key "github.com/ipfs/go-ipfs/blocks/key"
 
-	ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
+	"gx/ipfs/QmRg1gKTHzc3CZXSKzem8aR4E3TubFhbgXwfVuWnSK5CC5/go-metrics-interface"
 	lru "gx/ipfs/QmVYxfoJQiZijTgPNHCHgHELvQpbsJNTg6Crmc3dQkj3yy/golang-lru"
 	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
+	ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
 )
 
 type arccache struct {
 	arc        *lru.ARCCache
 	blockstore Blockstore
+
+	hits  metrics.Counter
+	total metrics.Counter
 }
 
-func arcCached(bs Blockstore, lruSize int) (*arccache, error) {
+func newARCCachedBS(ctx context.Context, bs Blockstore, lruSize int) (*arccache, error) {
 	arc, err := lru.NewARC(lruSize)
 	if err != nil {
 		return nil, err
 	}
+	c := &arccache{arc: arc, blockstore: bs}
+	c.hits = metrics.NewCtx(ctx, "arc.hits_total", "Number of ARC cache hits").Counter()
+	c.total = metrics.NewCtx(ctx, "arc_total", "Total number of ARC cache requests").Counter()
 
-	return &arccache{arc: arc, blockstore: bs}, nil
+	return c, nil
 }
 
 func (b *arccache) DeleteBlock(k key.Key) error {
@@ -42,6 +50,7 @@ func (b *arccache) DeleteBlock(k key.Key) error {
 // if ok == false has is inconclusive
 // if ok == true then has respons to question: is it contained
 func (b *arccache) hasCached(k key.Key) (has bool, ok bool) {
+	b.total.Inc()
 	if k == "" {
 		// Return cache invalid so the call to blockstore happens
 		// in case of invalid key and correct error is created.
@@ -50,6 +59,7 @@ func (b *arccache) hasCached(k key.Key) (has bool, ok bool) {
 
 	h, ok := b.arc.Get(k)
 	if ok {
+		b.hits.Inc()
 		return h.(bool), true
 	}
 	return false, false

blocks/blockstore/arc_cache_test.go

@@ -4,11 +4,11 @@ import (
 	"testing"
 
 	"github.com/ipfs/go-ipfs/blocks"
-	"github.com/ipfs/go-ipfs/blocks/key"
+	"gx/ipfs/Qmce4Y4zg3sYr7xKM5UueS67vhNni6EeWgCRnb7MbLJMew/go-key"
 
-	ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
-	syncds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/sync"
 	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
+	ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
+	syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
 )
 
 var exampleBlock = blocks.NewBlock([]byte("foo"))
@@ -140,7 +140,7 @@ func TestGetAndDeleteFalseShortCircuit(t *testing.T) {
 }
 
 func TestArcCreationFailure(t *testing.T) {
-	if arc, err := arcCached(nil, -1); arc != nil || err == nil {
+	if arc, err := newARCCachedBS(context.TODO(), nil, -1); arc != nil || err == nil {
 		t.Fatal("expected error and no cache")
 	}
 }

blocks/blockstore/blockstore.go

@@ -8,13 +8,13 @@ import (
 	"sync/atomic"
 
 	blocks "github.com/ipfs/go-ipfs/blocks"
-	key "github.com/ipfs/go-ipfs/blocks/key"
-	ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
-	dsns "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/namespace"
-	dsq "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/query"
 	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
 	mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
 	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
+	ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
+	dsns "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/namespace"
+	dsq "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/query"
+	key "gx/ipfs/Qmce4Y4zg3sYr7xKM5UueS67vhNni6EeWgCRnb7MbLJMew/go-key"
 )
 
 var log = logging.Logger("blockstore")
@@ -121,7 +121,7 @@ func (bs *blockstore) Put(block blocks.Block) (error, blocks.Block) {
 	if err == nil && exists {
 		return nil, nil // already stored.
 	}
-	return bs.datastore.Put(k, block.Data()), block
+	return bs.datastore.Put(k, block.RawData()), block
 }
 
 func (bs *blockstore) PutMany(blks []blocks.Block) (error, []blocks.Block) {
@@ -137,7 +137,7 @@ func (bs *blockstore) PutMany(blks []blocks.Block) (error, []blocks.Block) {
 			continue
 		}
 
-		err = t.Put(k, b.Data())
+		err = t.Put(k, b.RawData())
 		if err != nil {
 			return err, nil
 		}

blocks/blockstore/blockstore_test.go

@@ -5,14 +5,14 @@ import (
 	"fmt"
 	"testing"
 
-	ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
-	dsq "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/query"
-	ds_sync "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/sync"
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
 	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
+	ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
+	dsq "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/query"
+	ds_sync "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
 
 	blocks "github.com/ipfs/go-ipfs/blocks"
-	key "github.com/ipfs/go-ipfs/blocks/key"
+	key "gx/ipfs/Qmce4Y4zg3sYr7xKM5UueS67vhNni6EeWgCRnb7MbLJMew/go-key"
 )
 
 func TestGetWhenKeyNotPresent(t *testing.T) {
@@ -48,7 +48,7 @@ func TestPutThenGetBlock(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(block.Data(), blockFromBlockstore.Data()) {
+	if !bytes.Equal(block.RawData(), blockFromBlockstore.RawData()) {
 		t.Fail()
 	}
 }

blocks/blockstore/bloom_cache.go

@@ -2,12 +2,14 @@ package blockstore
 
 import (
 	"sync/atomic"
+	"time"
 
 	"github.com/ipfs/go-ipfs/blocks"
-	key "github.com/ipfs/go-ipfs/blocks/key"
+	key "gx/ipfs/Qmce4Y4zg3sYr7xKM5UueS67vhNni6EeWgCRnb7MbLJMew/go-key"
 
-	bloom "gx/ipfs/QmWQ2SJisXwcCLsUXLwYCKSfyExXjFRW2WbBH5sqCUnwX5/bbloom"
+	"gx/ipfs/QmRg1gKTHzc3CZXSKzem8aR4E3TubFhbgXwfVuWnSK5CC5/go-metrics-interface"
 	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
+	bloom "gx/ipfs/QmeiMCBkYHxkDkDfnDadzz4YxY5ruL5Pj499essE4vRsGM/bbloom"
 )
 
 // bloomCached returns Blockstore that caches Has requests using Bloom filter
@@ -18,9 +20,31 @@ func bloomCached(bs Blockstore, ctx context.Context, bloomSize, hashCount int) (
 		return nil, err
 	}
 	bc := &bloomcache{blockstore: bs, bloom: bl}
+	bc.hits = metrics.NewCtx(ctx, "bloom.hits_total",
+		"Number of cache hits in bloom cache").Counter()
+	bc.total = metrics.NewCtx(ctx, "bloom_total",
+		"Total number of requests to bloom cache").Counter()
+
 	bc.Invalidate()
 	go bc.Rebuild(ctx)
-
+	if metrics.Active() {
+		go func() {
+			fill := metrics.NewCtx(ctx, "bloom_fill_ratio",
+				"Ratio of bloom filter fullnes, (updated once a minute)").Gauge()
+
+			<-bc.rebuildChan
+			t := time.NewTicker(1 * time.Minute)
+			for {
+				select {
+				case <-ctx.Done():
+					t.Stop()
+					return
+				case <-t.C:
+					fill.Set(bc.bloom.FillRatio())
+				}
+			}
+		}()
+	}
 	return bc, nil
 }
 
@@ -33,8 +57,8 @@ type bloomcache struct {
 	blockstore  Blockstore
 
 	// Statistics
-	hits   uint64
-	misses uint64
+	hits  metrics.Counter
+	total metrics.Counter
 }
 
 func (b *bloomcache) Invalidate() {
@@ -84,6 +108,7 @@ func (b *bloomcache) DeleteBlock(k key.Key) error {
 // if ok == false has is inconclusive
 // if ok == true then has respons to question: is it contained
 func (b *bloomcache) hasCached(k key.Key) (has bool, ok bool) {
+	b.total.Inc()
 	if k == "" {
 		// Return cache invalid so call to blockstore
 		// in case of invalid key is forwarded deeper
@@ -92,6 +117,7 @@ func (b *bloomcache) hasCached(k key.Key) (has bool, ok bool) {
 	if b.BloomActive() {
 		blr := b.bloom.HasTS([]byte(k))
 		if blr == false { // not contained in bloom is only conclusive answer bloom gives
+			b.hits.Inc()
 			return false, true
 		}
 	}

blocks/blockstore/bloom_cache_test.go

@@ -8,10 +8,10 @@ import (
 
 	"github.com/ipfs/go-ipfs/blocks"
 
-	ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
-	dsq "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/query"
-	syncds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/sync"
 	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
+	ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
+	dsq "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/query"
+	syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
 )
 
 func testBloomCached(bs GCBlockstore, ctx context.Context) (*bloomcache, error) {