Merge pull request #223 from oneconcern/d20190625-cafsputrv--wip

cafs.Put result type

cmd/backup2blobs/cmd/file2blobs.go

@@ -54,7 +54,10 @@ func UploadToBlob(sourceStore storage.Store, backupStore storage.Store, cafs caf
  incC(errC)
  continue
  }
- size, key, _, duplicate, err := cafs.Put(context.Background(), reader)
+ putRes, err := cafs.Put(context.Background(), reader)
+ size := putRes.Written
+ key := putRes.Key
+ duplicate := putRes.Found
  if err != nil {
  logger.Error("Failed to upload blob", zap.String("file", file), zap.Error(err))
  incC(errC)

go.mod

@@ -10,8 +10,8 @@ require (
  github.com/container-storage-interface/spec v0.3.0
  github.com/docker/go-units v0.3.3
  github.com/go-chi/chi v4.0.2+incompatible
- github.com/gobuffalo/packd v0.2.0
- github.com/gobuffalo/packr/v2 v2.4.0
+ github.com/gobuffalo/packd v0.3.0
+ github.com/gobuffalo/packr/v2 v2.5.1
  github.com/hashicorp/go-immutable-radix v1.0.0
  github.com/hashicorp/golang-lru v0.5.0
  github.com/inconshreveable/mousetrap v1.0.0 // indirect
@@ -21,8 +21,11 @@ require (
  github.com/karrick/godirwalk v1.10.12
  github.com/kr/pty v1.1.5 // indirect
  github.com/kylelemons/godebug v1.1.0 // indirect
+ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 // indirect
+ github.com/markbates/safe v1.0.1 // indirect
  github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
  github.com/opentracing/opentracing-go v1.0.2
+ github.com/pkg/errors v0.8.1 // indirect
  github.com/segmentio/ksuid v1.0.2
  github.com/spf13/afero v1.2.2
  github.com/spf13/cobra v0.0.5
@@ -32,11 +35,10 @@ require (
  go.uber.org/atomic v1.3.2 // indirect
  go.uber.org/multierr v1.1.0 // indirect
  go.uber.org/zap v1.9.1
- golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 // indirect
  golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect
- golang.org/x/sys v0.0.0-20190621203818-d432491b9138 // indirect
+ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 // indirect
  golang.org/x/text v0.3.2 // indirect
- golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac // indirect
+ golang.org/x/tools v0.0.0-20190625160430-252024b82959 // indirect
  google.golang.org/api v0.2.0
  google.golang.org/grpc v1.19.0
  gopkg.in/yaml.v2 v2.2.2

go.sum

@@ -46,12 +46,18 @@ github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU=
 github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
 github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2 h1:8thhT+kUJMTMy3HlX4+y9Da+BNJck+p109tqqKp7WDs=
 github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/logger v1.0.0 h1:xw9Ko9EcC5iAFprrjJ6oZco9UpzS5MQ4jAwghsLHdy4=
+github.com/gobuffalo/logger v1.0.0/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs=
 github.com/gobuffalo/packd v0.1.0 h1:4sGKOD8yaYJ+dek1FDkwcxCHA40M4kfKgFHx8N2kwbU=
 github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
 github.com/gobuffalo/packd v0.2.0 h1:9Mu1TYbSfH75zQIQtZEgHEaYeRR4485U4T9N1OQYrXk=
 github.com/gobuffalo/packd v0.2.0/go.mod h1:k2CkHP3bjbqL2GwxwhxUy1DgnlbW644hkLC9iIUvZwY=
+github.com/gobuffalo/packd v0.3.0 h1:eMwymTkA1uXsqxS0Tpoop3Lc0u3kTfiMBE6nKtQU4g4=
+github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q=
 github.com/gobuffalo/packr/v2 v2.4.0 h1:FB8CdysPlcECofv3tpH21T4aJDWmv7NFq8m0X6JZMGc=
 github.com/gobuffalo/packr/v2 v2.4.0/go.mod h1:ra341gygw9/61nSjAbfwcwh8IrYL4WmR4IsPkPBhQiY=
+github.com/gobuffalo/packr/v2 v2.5.1 h1:TFOeY2VoGamPjQLiNDT3mn//ytzk236VMO2j7iHxJR4=
+github.com/gobuffalo/packr/v2 v2.5.1/go.mod h1:8f9c96ITobJlPzI44jj+4tHnEKNt0xXWSVlXRN9X1Iw=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -274,8 +280,11 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190426135247-a129542de9ae h1:mQLHiymj/JXKnnjc62tb7nD5pZLs940/sXJu+Xp3DBA=
 golang.org/x/sys v0.0.0-20190426135247-a129542de9ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190621203818-d432491b9138 h1:t8BZD9RDjkm9/h7yYN6kE8oaeov5r9aztkB7zKA5Tkg=
 golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -293,6 +302,9 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190613204242-ed0dc450797f/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624180213-70d37148ca0c/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190625160430-252024b82959 h1:VuVDXrx1TCtYGfuoKtAHdDqlDBF4b1ygX9yjx+WokdE=
+golang.org/x/tools v0.0.0-20190625160430-252024b82959/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

pkg/cafs/cafs.go

@@ -79,11 +79,18 @@ type hasOpts struct {
 // Option to configure content addressable FS components
 type Option func(*defaultFs)
 
+type PutRes struct {
+ Written int64
+ Key Key
+ Keys []byte
+ Found bool
+}
+
 // Fs implementations provide content-addressable filesystem operations
 type Fs interface {
  Get(context.Context, Key) (io.ReadCloser, error)
  GetAt(context.Context, Key) (io.ReaderAt, error)
- Put(context.Context, io.Reader) (int64, Key, []byte, bool, error)
+ Put(context.Context, io.Reader) (PutRes, error)
  Delete(context.Context, Key) error
  Clear(context.Context) error
  Keys(context.Context) ([]Key, error)
@@ -123,19 +130,19 @@ type defaultFs struct {
  readerConcurrentChunkWrites int
 }
 
-func (d *defaultFs) Put(ctx context.Context, src io.Reader) (int64, Key, []byte, bool, error) {
+func (d *defaultFs) Put(ctx context.Context, src io.Reader) (PutRes, error) {
  w := d.writer(d.prefix)
  defer w.Close()
  written, err := io.Copy(w, src)
  if err != nil {
- return 0, Key{}, nil, false, err
+ return PutRes{}, err
  }
  key, keys, err := w.Flush()
  if err != nil {
- return 0, Key{}, nil, false, err
+ return PutRes{}, err
  }
  if err = w.Close(); err != nil {
- return 0, Key{}, nil, false, err
+ return PutRes{}, err
  }
  destinations := make([]storage.MultiStoreUnit, 0)
 
@@ -149,9 +156,14 @@ func (d *defaultFs) Put(ctx context.Context, src io.Reader) (int64, Key, []byte,
  buffer := append(keys, key[:]...)
  err = storage.MultiPut(ctx, destinations, key.String(), buffer, storage.OverWrite)
  if err != nil {
- return 0, Key{}, nil, found, err
+ return PutRes{Found: found}, err
  }
- return written, key, keys, found, nil
+ return PutRes{
+ Written: written,
+ Key: key,
+ Keys: keys,
+ Found: found,
+ }, nil
 }
 
 func (d *defaultFs) Get(ctx context.Context, hash Key) (io.ReadCloser, error) {

pkg/cafs/cafs_test.go

@@ -46,6 +46,11 @@ func TestCAFS_Get(t *testing.T) {
 }
 
 func TestCAFS_Put(t *testing.T) {
+
+ putRes2Tuple := func(res PutRes) (int64, Key, []byte, bool) {
+ return res.Written, res.Key, res.Keys, res.Found
+ }
+
  td, err := ioutil.TempDir("", "tpt-cafs-put")
  require.NoError(t, err)
  defer os.RemoveAll(td)
@@ -61,7 +66,8 @@ func TestCAFS_Put(t *testing.T) {
  require.NoError(t, err)
  defer f.Close()
 
- written, rk, _, _, err := fs.Put(context.Background(), f)
+ putRes, err := fs.Put(context.Background(), f)
+ written, rk, _, _ := putRes2Tuple(putRes)
  require.NoError(t, err)
  fileInfo, err := f.Stat()
  require.NoError(t, err)
@@ -72,7 +78,8 @@ func TestCAFS_Put(t *testing.T) {
  f, err = os.Open(orig)
  require.NoError(t, err)
  defer f.Close()
- written, rk2, _, _, err := fs.Put(context.Background(), f)
+ putRes, err = fs.Put(context.Background(), f)
+ written, rk2, _, _ := putRes2Tuple(putRes)
  require.NoError(t, err)
  fileInfo, err = f.Stat()
  require.NoError(t, err)

pkg/cafs/fake_data_generator.go

@@ -74,9 +74,9 @@ func GenerateCAFSChunks(src string, fs Fs) (*Key, error) {
  }
  defer sourceFile.Close()
 
- _, key, _, _, err := fs.Put(context.Background(), sourceFile)
+ putRes, err := fs.Put(context.Background(), sourceFile)
  if err != nil {
  return nil, err
  }
- return &key, nil
+ return &putRes.Key, nil
 }

pkg/core/bundle_pack.go

@@ -109,7 +109,7 @@ func uploadBundleFile(
  defer func() {
  <-chans.concurrencyControl
  }()
- written, key, keys, duplicate, e := cafsArchive.Put(ctx, fileReader)
+ putRes, e := cafsArchive.Put(ctx, fileReader)
  if e != nil {
  select {
  case chans.error <- errorHit{
@@ -123,11 +123,11 @@ func uploadBundleFile(
 
  select {
  case chans.filePacked <- filePacked{
- hash: key.String(),
- keys: keys,
+ hash: putRes.Key.String(),
+ keys: putRes.Keys,
  name: file,
- size: uint64(written),
- duplicate: duplicate,
+ size: uint64(putRes.Written),
+ duplicate: putRes.Found,
  }:
  case <-chans.done:
  }

pkg/core/fs_rw_ops.go

@@ -624,7 +624,7 @@ func commitFileUpload(
  return
  }
  // written, key, keys, duplicate, err =
- written, key, _, _, err := caFs.Put(ctx, file)
+ putRes, err := caFs.Put(ctx, file)
  if err != nil {
  select {
  case chans.error <- err:
@@ -636,10 +636,10 @@ func commitFileUpload(
  return
  }
  be := model.BundleEntry{
- Hash: key.String(),
+ Hash: putRes.Key.String(),
  NameWithPath: uploadTask.name,
  FileMode: 0, // #TODO: #35 file mode support
- Size: uint64(written),
+ Size: uint64(putRes.Written),
  }
  select {
  case chans.bundleEntry <- be: