diff --git a/filestore.go b/filestore.go index be4d954..a9c36c5 100644 --- a/filestore.go +++ b/filestore.go @@ -19,7 +19,7 @@ import ( logging "github.com/ipfs/go-log" ) -var log = logging.Logger("filestore") +var logger = logging.Logger("filestore") var ErrFilestoreNotEnabled = errors.New("filestore is not enabled, see https://git.io/vNItf") var ErrUrlstoreNotEnabled = errors.New("urlstore is not enabled") @@ -86,7 +86,7 @@ func (f *Filestore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { // cant query leveldb concurrently b, err := f.fm.AllKeysChan(ctx) if err != nil { - log.Error("error querying filestore: ", err) + logger.Error("error querying filestore: ", err) return } diff --git a/fsrefstore.go b/fsrefstore.go index 19927e0..bc183fc 100644 --- a/fsrefstore.go +++ b/fsrefstore.go @@ -19,6 +19,7 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" posinfo "github.com/ipfs/go-ipfs-posinfo" + mh "github.com/multiformats/go-multihash" ) // FilestorePrefix identifies the key prefix for FileManager blocks. @@ -60,6 +61,8 @@ func NewFileManager(ds ds.Batching, root string) *FileManager { // AllKeysChan returns a channel from which to read the keys stored in // the FileManager. If the given context is cancelled the channel will be // closed. +// +// All CIDs returned are of type Raw. func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { q := dsq.Query{KeysOnly: true} @@ -78,14 +81,14 @@ func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { } k := ds.RawKey(v.Key) - c, err := dshelp.DsKeyToCid(k) + mhash, err := dshelp.DsKeyToMultihash(k) if err != nil { - log.Errorf("decoding cid from filestore: %s", err) + logger.Errorf("decoding cid from filestore: %s", err) continue } select { - case out <- c: + case out <- cid.NewCidV1(cid.Raw, mhash): case <-ctx.Done(): return } @@ -98,7 +101,7 @@ func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { // DeleteBlock deletes the reference-block from the underlying // datastore. It does not touch the referenced data. func (f *FileManager) DeleteBlock(c cid.Cid) error { - err := f.ds.Delete(dshelp.CidToDsKey(c)) + err := f.ds.Delete(dshelp.MultihashToDsKey(c.Hash())) if err == ds.ErrNotFound { return blockstore.ErrNotFound } @@ -110,11 +113,11 @@ func (f *FileManager) DeleteBlock(c cid.Cid) error { // block from the datastore. The second step uses the stored // path and offsets to read the raw block data directly from disk. func (f *FileManager) Get(c cid.Cid) (blocks.Block, error) { - dobj, err := f.getDataObj(c) + dobj, err := f.getDataObj(c.Hash()) if err != nil { return nil, err } - out, err := f.readDataObj(c, dobj) + out, err := f.readDataObj(c.Hash(), dobj) if err != nil { return nil, err } @@ -127,22 +130,22 @@ func (f *FileManager) Get(c cid.Cid) (blocks.Block, error) { // This method may successfully return the size even if returning the block // would fail because the associated file is no longer available. func (f *FileManager) GetSize(c cid.Cid) (int, error) { - dobj, err := f.getDataObj(c) + dobj, err := f.getDataObj(c.Hash()) if err != nil { return -1, err } return int(dobj.GetSize_()), nil } -func (f *FileManager) readDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) { +func (f *FileManager) readDataObj(m mh.Multihash, d *pb.DataObj) ([]byte, error) { if IsURL(d.GetFilePath()) { - return f.readURLDataObj(c, d) + return f.readURLDataObj(m, d) } - return f.readFileDataObj(c, d) + return f.readFileDataObj(m, d) } -func (f *FileManager) getDataObj(c cid.Cid) (*pb.DataObj, error) { - o, err := f.ds.Get(dshelp.CidToDsKey(c)) +func (f *FileManager) getDataObj(m mh.Multihash) (*pb.DataObj, error) { + o, err := f.ds.Get(dshelp.MultihashToDsKey(m)) switch err { case ds.ErrNotFound: return nil, blockstore.ErrNotFound @@ -164,7 +167,7 @@ func unmarshalDataObj(data []byte) (*pb.DataObj, error) { return &dobj, nil } -func (f *FileManager) readFileDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) { +func (f *FileManager) readFileDataObj(m mh.Multihash, d *pb.DataObj) ([]byte, error) { if !f.AllowFiles { return nil, ErrFilestoreNotEnabled } @@ -193,12 +196,15 @@ func (f *FileManager) readFileDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) return nil, &CorruptReferenceError{StatusFileError, err} } - outcid, err := c.Prefix().Sum(outbuf) + // Work with CIDs for this, as they are a nice wrapper and things + // will not break if multihashes underlying types change. + origCid := cid.NewCidV1(cid.Raw, m) + outcid, err := origCid.Prefix().Sum(outbuf) if err != nil { return nil, err } - if !c.Equals(outcid) { + if !origCid.Equals(outcid) { return nil, &CorruptReferenceError{StatusFileChanged, fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())} } @@ -207,7 +213,7 @@ func (f *FileManager) readFileDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) } // reads and verifies the block from URL -func (f *FileManager) readURLDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) { +func (f *FileManager) readURLDataObj(m mh.Multihash, d *pb.DataObj) ([]byte, error) { if !f.AllowUrls { return nil, ErrUrlstoreNotEnabled } @@ -237,12 +243,15 @@ func (f *FileManager) readURLDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) { } res.Body.Close() - outcid, err := c.Prefix().Sum(outbuf) + // Work with CIDs for this, as they are a nice wrapper and things + // will not break if multihashes underlying types change. + origCid := cid.NewCidV1(cid.Raw, m) + outcid, err := origCid.Prefix().Sum(outbuf) if err != nil { return nil, err } - if !c.Equals(outcid) { + if !origCid.Equals(outcid) { return nil, &CorruptReferenceError{StatusFileChanged, fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())} } @@ -255,7 +264,7 @@ func (f *FileManager) readURLDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) { func (f *FileManager) Has(c cid.Cid) (bool, error) { // NOTE: interesting thing to consider. Has doesnt validate the data. // So the data on disk could be invalid, and we could think we have it. - dsk := dshelp.CidToDsKey(c) + dsk := dshelp.MultihashToDsKey(c.Hash()) return f.ds.Has(dsk) } @@ -300,7 +309,7 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { return err } - return to.Put(dshelp.CidToDsKey(b.Cid()), data) + return to.Put(dshelp.MultihashToDsKey(b.Cid().Hash()), data) } // PutMany is like Put() but takes a slice of blocks instead, diff --git a/go.mod b/go.mod index ac9659d..096a5f1 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,11 @@ require ( github.com/gogo/protobuf v1.3.1 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-cid v0.0.5 - github.com/ipfs/go-datastore v0.4.1 - github.com/ipfs/go-ipfs-blockstore v0.1.3 - github.com/ipfs/go-ipfs-ds-help v0.0.1 + github.com/ipfs/go-datastore v0.4.4 + github.com/ipfs/go-ipfs-blockstore v1.0.0 + github.com/ipfs/go-ipfs-ds-help v1.0.0 github.com/ipfs/go-ipfs-posinfo v0.0.1 - github.com/ipfs/go-log v0.0.1 + github.com/ipfs/go-log v1.0.2 github.com/ipfs/go-merkledag v0.3.1 + github.com/multiformats/go-multihash v0.0.13 ) diff --git a/go.sum b/go.sum index 5f596ef..b1f4d4e 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,6 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJY github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -49,8 +47,6 @@ github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -75,8 +71,6 @@ github.com/ipfs/go-cid v0.0.2 h1:tuuKaZPU1M6HcejsO3AcYWW8sZ8MTvyxfc4uqB4eFE8= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= -github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-datastore v0.0.1 h1:AW/KZCScnBWlSb5JbnEnLKFWXL224LBEh/9KXXOrUms= @@ -85,12 +79,12 @@ github.com/ipfs/go-datastore v0.0.5 h1:q3OfiOZV5rlsK1H5V8benjeUApRfMGs4Mrhmr6Nri github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= -github.com/ipfs/go-datastore v0.1.1 h1:F4k0TkTAZGLFzBOrVKDAvch6JZtuN4NHkfdcEZL50aI= -github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU= github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.1 h1:W4ZfzyhNi3xmuU5dQhjfuRn/wFuqEE1KnOmmQiOevEY= github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8= +github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -99,10 +93,8 @@ github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xL github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi+MTo9O88= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= -github.com/ipfs/go-ipfs-blockstore v0.1.1 h1:+PAFREAlSxLs9IEtrRcnJ/DqWkGlDa+l547WFZnohNw= -github.com/ipfs/go-ipfs-blockstore v0.1.1/go.mod h1:8gZOgIN5e+Xdg2YSGdwTTRbguSVjYyosIDRQCY8E9QM= -github.com/ipfs/go-ipfs-blockstore v0.1.3 h1:Nc/Uwc8KZgo5PsMUZT/Zt7zc0u/s+b/3c64ChQNttHc= -github.com/ipfs/go-ipfs-blockstore v0.1.3/go.mod h1:iNWVBoSQ7eMcaGo8+L3pKZABGTdWcqj1/hpoUu5bDps= +github.com/ipfs/go-ipfs-blockstore v1.0.0 h1:pmFp5sFYsYVvMOp9X01AK3s85usVcLvkBTRsN6SnfUA= +github.com/ipfs/go-ipfs-blockstore v1.0.0/go.mod h1:knLVdhVU9L7CC4T+T4nvGdeUIPAXlnd9zmXfp+9MIjU= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -110,6 +102,8 @@ github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1I github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v0.0.1 h1:QBg+Ts2zgeemK/dB0saiF/ykzRGgfoFMT90Rzo0OnVU= github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= +github.com/ipfs/go-ipfs-ds-help v1.0.0 h1:bEQ8hMGs80h0sR8O4tfDgV6B01aaF9qeTrujrTLYV3g= +github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE= github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6rY30GUu7G6LUfpMvM= github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew= @@ -122,8 +116,6 @@ github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRD github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-ipld-cbor v0.0.2 h1:amzFztBQQQ69UA5+f7JRfoXF/z2l//MGfEDHVkS20+s= -github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3 h1:ENsxvybwkmke7Z/QJOmeJfoguj6GH3Y0YOaGrfy9Q0I= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-format v0.0.1 h1:HCu4eB/Gh+KD/Q0M8u888RFkorTWNIL3da4oc5dwc80= @@ -132,10 +124,10 @@ github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2 github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= -github.com/ipfs/go-merkledag v0.1.0 h1:CAEXjRFEDPvealQj3TgEjV1IJckwjvmxAqtq5QSXJrg= -github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= -github.com/ipfs/go-merkledag v0.2.4 h1:ZSHQSe9BENfixUjT+MaLeHEeZGxrZQfgo3KT3SLosF8= -github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-log v1.0.2 h1:s19ZwJxH8rPWzypjcDpqPLIyV7BnbLqvpli3iZoqYK0= +github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= +github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw= +github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-merkledag v0.3.1 h1:3UqWINBEr3/N+r6OwgFXAddDP/8zpQX/8J7IGVOCqRQ= github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= @@ -324,6 +316,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= @@ -337,6 +331,12 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -376,6 +376,7 @@ google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoA gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/util.go b/util.go index bfb240c..dc860f7 100644 --- a/util.go +++ b/util.go @@ -11,6 +11,7 @@ import ( dsq "github.com/ipfs/go-datastore/query" blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" + mh "github.com/multiformats/go-multihash" ) // Status is used to identify the state of the block data referenced @@ -86,7 +87,7 @@ func (r *ListRes) FormatLong(enc func(cid.Cid) string) string { // List does not verify that the reference is valid or whether the // raw data is accesible. See Verify(). func List(fs *Filestore, key cid.Cid) *ListRes { - return list(fs, false, key) + return list(fs, false, key.Hash()) } // ListAll returns a function as an iterator which, once invoked, returns @@ -105,7 +106,7 @@ func ListAll(fs *Filestore, fileOrder bool) (func() *ListRes, error) { // Verify makes sure that the reference is valid and the block data can be // read. func Verify(fs *Filestore, key cid.Cid) *ListRes { - return list(fs, true, key) + return list(fs, true, key.Hash()) } // VerifyAll returns a function as an iterator which, once invoked, @@ -119,7 +120,7 @@ func VerifyAll(fs *Filestore, fileOrder bool) (func() *ListRes, error) { return listAll(fs, true) } -func list(fs *Filestore, verify bool, key cid.Cid) *ListRes { +func list(fs *Filestore, verify bool, key mh.Multihash) *ListRes { dobj, err := fs.fm.getDataObj(key) if err != nil { return mkListRes(key, nil, err) @@ -138,34 +139,34 @@ func listAll(fs *Filestore, verify bool) (func() *ListRes, error) { } return func() *ListRes { - cid, dobj, err := next(qr) + mhash, dobj, err := next(qr) if dobj == nil && err == nil { return nil } else if err == nil && verify { - _, err = fs.fm.readDataObj(cid, dobj) + _, err = fs.fm.readDataObj(mhash, dobj) } - return mkListRes(cid, dobj, err) + return mkListRes(mhash, dobj, err) }, nil } -func next(qr dsq.Results) (cid.Cid, *pb.DataObj, error) { +func next(qr dsq.Results) (mh.Multihash, *pb.DataObj, error) { v, ok := qr.NextSync() if !ok { - return cid.Cid{}, nil, nil + return nil, nil, nil } k := ds.RawKey(v.Key) - c, err := dshelp.DsKeyToCid(k) + mhash, err := dshelp.DsKeyToMultihash(k) if err != nil { - return cid.Cid{}, nil, fmt.Errorf("decoding cid from filestore: %s", err) + return nil, nil, fmt.Errorf("decoding multihash from filestore: %s", err) } dobj, err := unmarshalDataObj(v.Value) if err != nil { - return c, nil, err + return mhash, nil, err } - return c, dobj, nil + return mhash, dobj, nil } func listAllFileOrder(fs *Filestore, verify bool) (func() *ListRes, error) { @@ -206,12 +207,12 @@ func listAllFileOrder(fs *Filestore, verify bool) (func() *ListRes, error) { } v := entries[i] i++ - // attempt to convert the datastore key to a CID, + // attempt to convert the datastore key to a Multihash, // store the error but don't use it yet - cid, keyErr := dshelp.DsKeyToCid(ds.RawKey(v.dsKey)) + mhash, keyErr := dshelp.DsKeyToMultihash(ds.RawKey(v.dsKey)) // first if they listRes already had an error return that error if v.err != nil { - return mkListRes(cid, nil, v.err) + return mkListRes(mhash, nil, v.err) } // now reconstruct the DataObj dobj := pb.DataObj{ @@ -222,14 +223,14 @@ func listAllFileOrder(fs *Filestore, verify bool) (func() *ListRes, error) { // now if we could not convert the datastore key return that // error if keyErr != nil { - return mkListRes(cid, &dobj, keyErr) + return mkListRes(mhash, &dobj, keyErr) } // finally verify the dataobj if requested var err error if verify { - _, err = fs.fm.readDataObj(cid, &dobj) + _, err = fs.fm.readDataObj(mhash, &dobj) } - return mkListRes(cid, &dobj, err) + return mkListRes(mhash, &dobj, err) }, nil } @@ -255,7 +256,7 @@ func (l listEntries) Less(i, j int) bool { return l[i].filePath < l[j].filePath } -func mkListRes(c cid.Cid, d *pb.DataObj, err error) *ListRes { +func mkListRes(m mh.Multihash, d *pb.DataObj, err error) *ListRes { status := StatusOk errorMsg := "" if err != nil { @@ -268,6 +269,9 @@ func mkListRes(c cid.Cid, d *pb.DataObj, err error) *ListRes { } errorMsg = err.Error() } + + c := cid.NewCidV1(cid.Raw, m) + if d == nil { return &ListRes{ Status: status,