From a414feb474f36333ac3da95fd527bda53258bcce Mon Sep 17 00:00:00 2001 From: Gal Topper Date: Tue, 27 Jul 2021 16:56:17 +0300 Subject: [PATCH] Fix paging when listing a v3io directory. (#2) --- backends/v3io.go | 86 ++++++++++++++++++++++++++---------------------- go.mod | 6 ++-- go.sum | 35 +++++++++++++++----- 3 files changed, 75 insertions(+), 52 deletions(-) diff --git a/backends/v3io.go b/backends/v3io.go index 87911b1..d14c214 100644 --- a/backends/v3io.go +++ b/backends/v3io.go @@ -13,8 +13,6 @@ import ( "time" ) -const defaultWorkers = 0 - const ( V3ioPathEnvironmentVariable = "V3IO_API" V3ioUserEnvironmentVariable = "V3IO_USERNAME" @@ -81,54 +79,62 @@ func (c *V3ioClient) ListDir(fileChan chan *FileDetails, task *ListDirTask, summ func (c *V3ioClient) getDir(path string, fileChan chan *FileDetails, summary *ListSummary) error { - resp, err := c.container.GetContainerContentsSync(&v3io.GetContainerContentsInput{Path: path}) - if err != nil { - c.logger.ErrorWith("ListBucket failed", "endpoint", c.params.Endpoint, "container", - c.params.Bucket, "path", path) - return errors.Wrap(err, "failed v3io ListBucket") - } - result := resp.Output.(*v3io.GetContainerContentsOutput) + req := v3io.GetContainerContentsInput{Path: path} + for { + resp, err := c.container.GetContainerContentsSync(&req) + if err != nil { + c.logger.ErrorWith("ListBucket failed", "endpoint", c.params.Endpoint, "container", + c.params.Bucket, "path", path) + return errors.Wrap(err, "failed v3io ListBucket") + } + result := resp.Output.(*v3io.GetContainerContentsOutput) - for _, obj := range result.Contents { + for _, obj := range result.Contents { - if strings.HasSuffix(obj.Key, "/") { - continue - } + if strings.HasSuffix(obj.Key, "/") { + continue + } - t, err := time.Parse(time.RFC3339, obj.LastModified) - if err != nil { - return errors.Wrap(err, "Invalid object time string - not an RFC 3339 time format.") - } - var size int64 - if obj.Size != nil { - size = int64(*obj.Size) - } - _, name := filepath.Split(obj.Key) + t, err := time.Parse(time.RFC3339, obj.LastModified) + if err != nil { + return errors.Wrap(err, "Invalid object time string - not an RFC 3339 time format.") + } + var size int64 + if obj.Size != nil { + size = int64(*obj.Size) + } + _, name := filepath.Split(obj.Key) - if !IsMatch(c.task, name, t, size) { - continue - } + if !IsMatch(c.task, name, t, size) { + continue + } - c.logger.DebugWith("List dir:", "key", obj.Key, "modified", obj.LastModified, "size", obj.Size) - fileDetails := &FileDetails{ - Key: obj.Key, Size: size, //Mtime: obj.LastModified, - } + c.logger.DebugWith("List dir:", "key", obj.Key, "modified", obj.LastModified, "size", obj.Size) + fileDetails := &FileDetails{ + Key: obj.Key, Size: size, //Mtime: obj.LastModified, + } - summary.TotalBytes += size - summary.TotalFiles += 1 - fileChan <- fileDetails - } + summary.TotalBytes += size + summary.TotalFiles += 1 + fileChan <- fileDetails + } - if c.task.Recursive { - for _, val := range result.CommonPrefixes { - _, name := filepath.Split(val.Prefix[0 : len(val.Prefix)-1]) - if c.task.Hidden || !strings.HasPrefix(name, ".") { - err = c.getDir(val.Prefix, fileChan, summary) - if err != nil { - return err + if c.task.Recursive { + for _, val := range result.CommonPrefixes { + _, name := filepath.Split(val.Prefix[0 : len(val.Prefix)-1]) + if c.task.Hidden || !strings.HasPrefix(name, ".") { + err = c.getDir(val.Prefix, fileChan, summary) + if err != nil { + return err + } } } } + + if !result.IsTruncated { + break + } + req.Marker = result.NextMarker } return nil } diff --git a/go.mod b/go.mod index 2ceb08d..eccfd9f 100644 --- a/go.mod +++ b/go.mod @@ -12,10 +12,10 @@ require ( github.com/pkg/errors v0.8.1 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect github.com/stretchr/testify v1.4.0 - github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04ff8591cb4b22270d598b36fc0d49a + github.com/tinylib/msgp v1.1.6 // indirect + github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04f github.com/valyala/fasthttp v1.4.0 // indirect go.uber.org/atomic v1.4.0 // indirect - golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 - golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 gopkg.in/ini.v1 v1.46.0 // indirect ) diff --git a/go.sum b/go.sum index e6f2c4b..62a24fd 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -34,6 +33,8 @@ github.com/nuclio/zap v0.0.2 h1:rY5PkMOl8CTkqRqIPuxziBiKK6Mq/8oEurfgRnNtqf0= github.com/nuclio/zap v0.0.2/go.mod h1:SUxPsgePvlyjx6c5MtGdB50pf0IQThtlyLwISLboeuc= github.com/pavius/zap v1.4.2-0.20180228181622-8d52692529b8 h1:WqLgmr/wj9TO5Sc6oYPQRAJBxuHE0NTeuVeFnT+FZVo= github.com/pavius/zap v1.4.2-0.20180228181622-8d52692529b8/go.mod h1:6FWOCx06uh50GClv8S2cfk3asqTJs3qq3ZNRtLZE77I= +github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -46,34 +47,50 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04ff8591cb4b22270d598b36fc0d49a h1:3xiuabjd2Qy1auDKeqJfkaASmsn0UEtrtwUeJKYZ5O4= -github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04ff8591cb4b22270d598b36fc0d49a/go.mod h1:IFb6dJiyvJnOjXUoCoPJ5UViaYjgVYmqJb4fD1qDeLk= +github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= +github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= +github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04f h1:idkl60lTFFDyo2pGm8fEdv4anon/E0UFrWBttWl7yz0= +github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04f/go.mod h1:IFb6dJiyvJnOjXUoCoPJ5UViaYjgVYmqJb4fD1qDeLk= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/fasthttp v1.4.0 h1:PuaTGZIw3mjYhhhbVbCQp8aciRZN9YdoB7MGX9Ko76A= github.com/valyala/fasthttp v1.4.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.46.0 h1:VeDZbLYGaupuvIrsYCEOe/L/2Pcs5n7hdO1ZTjporag=