From 870fc826bec3d70feb4564df645dca8c5028375f Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 19:08:09 +0100 Subject: [PATCH 1/4] Roundtrip test for partial file reads and traversals --- go.mod | 2 +- go.sum | 6 ++ test/partial_file_access_test.go | 108 +++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 test/partial_file_access_test.go diff --git a/go.mod b/go.mod index f03e414..279ca3f 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-unixfs v0.3.1 github.com/ipld/go-car/v2 v2.1.1 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.14.4 + github.com/ipld/go-ipld-prime v0.14.5-0.20220303180638-1a0310848f49 github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c github.com/multiformats/go-multihash v0.1.0 github.com/spaolacci/murmur3 v1.1.0 diff --git a/go.sum b/go.sum index 11fe1ed..878882f 100644 --- a/go.sum +++ b/go.sum @@ -125,6 +125,7 @@ github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -186,6 +187,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -349,6 +351,10 @@ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHt github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.14.4 h1:bqhmume8+nbNsX4/+J6eohktfZHAI8GKrF3rQ0xgOyc= github.com/ipld/go-ipld-prime v0.14.4/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.14.5-0.20220303150028-e0f4c23806e3 h1:D7Flnur2X2UWyoAe8StujzE01mSPLe8kiMCvG5d2UAM= +github.com/ipld/go-ipld-prime v0.14.5-0.20220303150028-e0f4c23806e3/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= +github.com/ipld/go-ipld-prime v0.14.5-0.20220303180638-1a0310848f49 h1:NGnRDbc2o4Vy/CjiY9KCPg0pSXcdipf8tTSPUiPPpDs= +github.com/ipld/go-ipld-prime v0.14.5-0.20220303180638-1a0310848f49/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= diff --git a/test/partial_file_access_test.go b/test/partial_file_access_test.go new file mode 100644 index 0000000..fffe74f --- /dev/null +++ b/test/partial_file_access_test.go @@ -0,0 +1,108 @@ +package test + +import ( + "bytes" + "context" + "io" + "testing" + + u "github.com/ipfs/go-ipfs-util" + "github.com/ipfs/go-unixfsnode/data/builder" + "github.com/ipfs/go-unixfsnode/file" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipld/go-ipld-prime/traversal" + sb "github.com/ipld/go-ipld-prime/traversal/selector/builder" +) + +func TestPartialFileAccess(t *testing.T) { + buf := make([]byte, 10*1024*1024) + u.NewSeededRand(0xdeadbeef).Read(buf) + r := bytes.NewReader(buf) + + ls := cidlink.DefaultLinkSystem() + storage := cidlink.Memory{} + ls.StorageReadOpener = storage.OpenRead + ls.StorageWriteOpener = storage.OpenWrite + + f, _, err := builder.BuildUnixFSFile(r, "", &ls) + if err != nil { + t.Fatal(err) + } + + // get back the root node substrate from the link at the top of the builder. + fr, err := ls.Load(ipld.LinkContext{}, f, dagpb.Type.PBNode) + if err != nil { + t.Fatal(err) + } + + ufn, err := file.NewUnixFSFile(context.Background(), fr, &ls) + if err != nil { + t.Fatal(err) + } + + openedLinks := []ipld.Link{} + ls.StorageReadOpener = func(lc linking.LinkContext, l datamodel.Link) (io.Reader, error) { + openedLinks = append(openedLinks, l) + return storage.OpenRead(lc, l) + } + + // read back out the file. + out, err := ufn.AsBytes() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(out, buf) { + t.Fatal("Not equal") + } + + fullLen := len(openedLinks) + + openedLinks = []ipld.Link{} + + partial, err := ufn.(datamodel.LargeBytesNode).AsLargeBytes() + if err != nil { + t.Fatal(err) + } + half := make([]byte, len(buf)/2) + if _, err := partial.Read(half); err != nil { + t.Fatal(err) + } + if len(openedLinks) >= fullLen { + t.Fatal("should not have accessed full file on a partial read.") + } + + openedLinks = []ipld.Link{} + + prog := traversal.Progress{ + Cfg: &traversal.Config{ + LinkSystem: ls, + }, + } + sb := sb.NewSelectorSpecBuilder(basicnode.Prototype.Any) + ss := sb.MatcherSubset(5*1024*1024, 6*1024*1024) + sel, err := ss.Selector() + if err != nil { + t.Fatal(err) + } + + if err := prog.WalkMatching(ufn, sel, func(_ traversal.Progress, n datamodel.Node) error { + b, err := n.AsBytes() + if err != nil { + t.Fatal(err) + } + if len(b) != 1024*1024 { + t.Fatalf("wrong length: %d", len(b)) + } + return nil + }); err != nil { + t.Fatal(err) + } + if len(openedLinks) >= fullLen { + t.Fatal("should not have accessed full file on a partial traversal.") + } +} From e00f90a2aaf9cba3b57e8da8978ddf91d16434b2 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 19:13:12 +0100 Subject: [PATCH 2/4] add doc to make linter happier --- test/doc.go | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 test/doc.go diff --git a/test/doc.go b/test/doc.go new file mode 100644 index 0000000..cecd77b --- /dev/null +++ b/test/doc.go @@ -0,0 +1,4 @@ +// Package test provides ADL testing of the ipld specification around +// * traversal making use of match subsets +// * largeByteNode readers +package test From 75d51255cc4b8f663ee8162ffd8cb13e2b871f87 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 19:24:33 +0100 Subject: [PATCH 3/4] tidy --- go.sum | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/go.sum b/go.sum index 878882f..05f05f7 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnXyBns= github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -185,8 +185,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -349,10 +349,6 @@ github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9 github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= -github.com/ipld/go-ipld-prime v0.14.4 h1:bqhmume8+nbNsX4/+J6eohktfZHAI8GKrF3rQ0xgOyc= -github.com/ipld/go-ipld-prime v0.14.4/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= -github.com/ipld/go-ipld-prime v0.14.5-0.20220303150028-e0f4c23806e3 h1:D7Flnur2X2UWyoAe8StujzE01mSPLe8kiMCvG5d2UAM= -github.com/ipld/go-ipld-prime v0.14.5-0.20220303150028-e0f4c23806e3/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= github.com/ipld/go-ipld-prime v0.14.5-0.20220303180638-1a0310848f49 h1:NGnRDbc2o4Vy/CjiY9KCPg0pSXcdipf8tTSPUiPPpDs= github.com/ipld/go-ipld-prime v0.14.5-0.20220303180638-1a0310848f49/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= From 30bbc6052f3970dcc0ac5d522b6bd865a612bb3b Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 9 Mar 2022 14:09:17 +0000 Subject: [PATCH 4/4] fix(deps): update to tagged ipld --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 279ca3f..5ee787e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-unixfs v0.3.1 github.com/ipld/go-car/v2 v2.1.1 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.14.5-0.20220303180638-1a0310848f49 + github.com/ipld/go-ipld-prime v0.16.0 github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c github.com/multiformats/go-multihash v0.1.0 github.com/spaolacci/murmur3 v1.1.0 diff --git a/go.sum b/go.sum index 05f05f7..6c414e4 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9 github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= -github.com/ipld/go-ipld-prime v0.14.5-0.20220303180638-1a0310848f49 h1:NGnRDbc2o4Vy/CjiY9KCPg0pSXcdipf8tTSPUiPPpDs= -github.com/ipld/go-ipld-prime v0.14.5-0.20220303180638-1a0310848f49/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= +github.com/ipld/go-ipld-prime v0.16.0 h1:RS5hhjB/mcpeEPJvfyj0qbOj/QL+/j05heZ0qa97dVo= +github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=