diff --git a/core/commands/refs.go b/core/commands/refs.go index 4a7285cc95c..3a7d51e7b5f 100644 --- a/core/commands/refs.go +++ b/core/commands/refs.go @@ -134,7 +134,7 @@ var RefsLocalCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List all local references.", ShortDescription: ` -Displays the hashes of all local objects. +Displays the hashes of all local objects. NOTE: This treats all local objects as "raw blocks" and returns CIDv1-Raw CIDs. `, }, diff --git a/core/node/storage.go b/core/node/storage.go index f1d9495fe53..a6c8a46ef1c 100644 --- a/core/node/storage.go +++ b/core/node/storage.go @@ -14,7 +14,6 @@ import ( "github.com/ipfs/go-filestore" "github.com/ipfs/go-ipfs/core/node/helpers" "github.com/ipfs/go-ipfs/repo" - "github.com/ipfs/go-ipfs/thirdparty/cidv0v1" "github.com/ipfs/go-ipfs/thirdparty/verifbs" ) @@ -61,7 +60,6 @@ func BaseBlockstoreCtor(cacheOpts blockstore.CacheOpts, nilRepo bool, hashOnRead } bs = blockstore.NewIdStore(bs) - bs = cidv0v1.NewBlockstore(bs) if hashOnRead { // TODO: review: this is how it was done originally, is there a reason we can't just pass this directly? bs.HashOnRead(true) diff --git a/gc/gc.go b/gc/gc.go index 6b44771d4e9..86eec0706b8 100644 --- a/gc/gc.go +++ b/gc/gc.go @@ -28,6 +28,16 @@ type Result struct { Error error } +// converts a set of CIDs with different codecs to a set of CIDs with the raw codec. +func toRawCids(set *cid.Set) *cid.Set { + newSet := cid.NewSet() + set.ForEach(func(c cid.Cid) error { + newSet.Add(cid.NewCidV1(cid.Raw, c.Hash())) + return nil + }) + return newSet +} + // GC performs a mark and sweep garbage collection of the blocks in the blockstore // first, it creates a 'marked' set and adds to it the following: // - all recursively pinned blocks, plus all of their descendants (recursively) @@ -65,6 +75,8 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn } return } + // The blockstore reports raw blocks. We need to remove the codecs from the CIDs. + gcs = toRawCids(gcs) emark.Append(logging.LoggableMap{ "blackSetSize": fmt.Sprintf("%d", gcs.Len()), }) @@ -90,6 +102,8 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn if !ok { break loop } + // NOTE: assumes that all CIDs returned by the keychan are _raw_ CIDv1 CIDs. + // This means we keep the block as long as we want it somewhere (CIDv1, CIDv0, Raw, other...). if !gcs.Has(k) { err := bs.DeleteBlock(k) removed++ @@ -170,7 +184,9 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots for _, c := range roots { // Walk recursively walks the dag and adds the keys to the given set - err := dag.Walk(ctx, verifyGetLinks, c, set.Visit, dag.Concurrent()) + err := dag.Walk(ctx, verifyGetLinks, c, func(k cid.Cid) bool { + return set.Visit(toCidV1(k)) + }, dag.Concurrent()) if err != nil { err = verboseCidError(err) @@ -181,6 +197,14 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots return nil } +// toCidV1 converts any CIDv0s to CIDv1s. +func toCidV1(c cid.Cid) cid.Cid { + if c.Version() == 0 { + return cid.NewCidV1(c.Type(), c.Hash()) + } + return c +} + // ColoredSet computes the set of nodes in the graph that are pinned by the // pins in the given pinner. func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffortRoots []cid.Cid, output chan<- Result) (*cid.Set, error) { @@ -241,7 +265,7 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo return nil, err } for _, k := range dkeys { - gcs.Add(k) + gcs.Add(toCidV1(k)) } ikeys, err := pn.InternalPins(ctx) diff --git a/go.mod b/go.mod index 4e471b7193b..9010b8c6a2e 100644 --- a/go.mod +++ b/go.mod @@ -27,11 +27,11 @@ require ( github.com/ipfs/go-ds-measure v0.1.0 github.com/ipfs/go-filestore v0.0.3 github.com/ipfs/go-fs-lock v0.0.3 - github.com/ipfs/go-ipfs-blockstore v0.1.1 + github.com/ipfs/go-ipfs-blockstore v0.1.2-0.20200107182344-e0992be5aac7 github.com/ipfs/go-ipfs-chunker v0.0.3 github.com/ipfs/go-ipfs-cmds v0.1.1 github.com/ipfs/go-ipfs-config v0.2.0 - github.com/ipfs/go-ipfs-ds-help v0.0.1 + github.com/ipfs/go-ipfs-ds-help v0.0.2-0.20200107180048-11890cc86e62 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-files v0.0.6 @@ -64,8 +64,8 @@ require ( github.com/libp2p/go-libp2p-core v0.3.0 github.com/libp2p/go-libp2p-discovery v0.2.0 github.com/libp2p/go-libp2p-http v0.1.4 - github.com/libp2p/go-libp2p-kad-dht v0.4.1 - github.com/libp2p/go-libp2p-kbucket v0.2.2 + github.com/libp2p/go-libp2p-kad-dht v0.5.0 + github.com/libp2p/go-libp2p-kbucket v0.2.3 github.com/libp2p/go-libp2p-loggables v0.1.0 github.com/libp2p/go-libp2p-mplex v0.2.1 github.com/libp2p/go-libp2p-peerstore v0.1.4 diff --git a/go.sum b/go.sum index b3afe247eb1..ae7e4311d46 100644 --- a/go.sum +++ b/go.sum @@ -200,8 +200,10 @@ github.com/ipfs/go-fs-lock v0.0.3/go.mod h1:j2WZOYsBPcZG7UJUh1tE61Nj4eZ1vImU69AQ github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi+MTo9O88= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= -github.com/ipfs/go-ipfs-blockstore v0.1.1 h1:+PAFREAlSxLs9IEtrRcnJ/DqWkGlDa+l547WFZnohNw= -github.com/ipfs/go-ipfs-blockstore v0.1.1/go.mod h1:8gZOgIN5e+Xdg2YSGdwTTRbguSVjYyosIDRQCY8E9QM= +github.com/ipfs/go-ipfs-blockstore v0.1.2-0.20200107173559-39b8fa892f23 h1:1YUM3VG1zFmWZJb0VqKpbqvtIHzZWrJ+u2XzyAxa2mI= +github.com/ipfs/go-ipfs-blockstore v0.1.2-0.20200107173559-39b8fa892f23/go.mod h1:KCm3qqlyBnbrIkkEHYiHO6Nwxse/sM7GjHZ2n6+27UQ= +github.com/ipfs/go-ipfs-blockstore v0.1.2-0.20200107182344-e0992be5aac7 h1:Ad79IyU49J46ATIm50glAqeIcJJmfHzw3LXXoMlwYPY= +github.com/ipfs/go-ipfs-blockstore v0.1.2-0.20200107182344-e0992be5aac7/go.mod h1:TByk/ha1EZ64DGA2FrXH/8HPZvk61IE89W81Iq+RC48= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= @@ -217,6 +219,10 @@ github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1I github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v0.0.1 h1:QBg+Ts2zgeemK/dB0saiF/ykzRGgfoFMT90Rzo0OnVU= github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= +github.com/ipfs/go-ipfs-ds-help v0.0.2-0.20200107173535-071cf2672860 h1:1f6exC5H9UZLT6tIBeRKfy1prnO2Fq1gAVEAcvngQGk= +github.com/ipfs/go-ipfs-ds-help v0.0.2-0.20200107173535-071cf2672860/go.mod h1:zjN0aB3d6VyzJTSvcylSYxo+LSR/ELEfNnQM8p/vwc8= +github.com/ipfs/go-ipfs-ds-help v0.0.2-0.20200107180048-11890cc86e62 h1:ib57ZzQCRZ/scmBO32DqWThjwpTqBW/fGqurYDE/rFA= +github.com/ipfs/go-ipfs-ds-help v0.0.2-0.20200107180048-11890cc86e62/go.mod h1:zjN0aB3d6VyzJTSvcylSYxo+LSR/ELEfNnQM8p/vwc8= github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6rY30GUu7G6LUfpMvM= github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew= @@ -410,10 +416,10 @@ github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM github.com/libp2p/go-libp2p-interface-connmgr v0.0.4/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= github.com/libp2p/go-libp2p-interface-connmgr v0.0.5/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k= -github.com/libp2p/go-libp2p-kad-dht v0.4.1 h1:N++/IVD7KemtNqwoqBLsmpc1PxROW1cxi81ja+wsJCg= -github.com/libp2p/go-libp2p-kad-dht v0.4.1/go.mod h1:Qf5Ddk5Csgi657ja2u5+NugbWz/QOVeVfrM1HTRDcfQ= -github.com/libp2p/go-libp2p-kbucket v0.2.2 h1:Jg/JUbQix6mvTnj+86FasRqkh01JFQNrN+H26Gxxsg0= -github.com/libp2p/go-libp2p-kbucket v0.2.2/go.mod h1:opWrBZSWnBYPc315q497huxY3sz1t488X6OiXUEYWKA= +github.com/libp2p/go-libp2p-kad-dht v0.5.0 h1:kDMtCftpQOL2s84/dZmw5z4NmBe6ByeDLKpcn6TcyxU= +github.com/libp2p/go-libp2p-kad-dht v0.5.0/go.mod h1:42YDfiKXzIgaIexiEQ3rKZbVPVPziLOyHpXbOCVd814= +github.com/libp2p/go-libp2p-kbucket v0.2.3 h1:XtNfN4WUy0cfeJoJgWCf1lor4Pp3kBkFJ9vQ+Zs+VUM= +github.com/libp2p/go-libp2p-kbucket v0.2.3/go.mod h1:opWrBZSWnBYPc315q497huxY3sz1t488X6OiXUEYWKA= github.com/libp2p/go-libp2p-loggables v0.0.1/go.mod h1:lDipDlBNYbpyqyPX/KcoO+eq0sJYEVR2JgOexcivchg= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= diff --git a/thirdparty/cidv0v1/blockstore.go b/thirdparty/cidv0v1/blockstore.go deleted file mode 100644 index 2f098dff9fe..00000000000 --- a/thirdparty/cidv0v1/blockstore.go +++ /dev/null @@ -1,87 +0,0 @@ -package cidv0v1 - -import ( - blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" - bs "github.com/ipfs/go-ipfs-blockstore" - mh "github.com/multiformats/go-multihash" -) - -type blockstore struct { - bs.Blockstore -} - -func NewBlockstore(b bs.Blockstore) bs.Blockstore { - return &blockstore{b} -} - -func (b *blockstore) Has(c cid.Cid) (bool, error) { - have, err := b.Blockstore.Has(c) - if have || err != nil { - return have, err - } - c1 := tryOtherCidVersion(c) - if !c1.Defined() { - return false, nil - } - return b.Blockstore.Has(c1) -} - -func (b *blockstore) Get(c cid.Cid) (blocks.Block, error) { - block, err := b.Blockstore.Get(c) - if err == nil { - return block, nil - } - if err != bs.ErrNotFound { - return nil, err - } - c1 := tryOtherCidVersion(c) - if !c1.Defined() { - return nil, bs.ErrNotFound - } - block, err = b.Blockstore.Get(c1) - if err != nil { - return nil, err - } - // modify block so it has the original CID - block, err = blocks.NewBlockWithCid(block.RawData(), c) - if err != nil { - return nil, err - } - // insert the block with the original CID to avoid problems - // with pinning - err = b.Blockstore.Put(block) - if err != nil { - return nil, err - } - return block, nil -} - -func (b *blockstore) GetSize(c cid.Cid) (int, error) { - size, err := b.Blockstore.GetSize(c) - if err == nil { - return size, nil - } - if err != bs.ErrNotFound { - return -1, err - } - c1 := tryOtherCidVersion(c) - if !c1.Defined() { - return -1, bs.ErrNotFound - } - return b.Blockstore.GetSize(c1) -} - -func tryOtherCidVersion(c cid.Cid) cid.Cid { - prefix := c.Prefix() - if prefix.Codec != cid.DagProtobuf || prefix.MhType != mh.SHA2_256 || prefix.MhLength != 32 { - return cid.Undef - } - var c1 cid.Cid - if prefix.Version == 0 { - c1 = cid.NewCidV1(cid.DagProtobuf, c.Hash()) - } else { - c1 = cid.NewCidV0(c.Hash()) - } - return c1 -}