diff --git a/cmd/client-fs.go b/cmd/client-fs.go index b49f528786..df72743e70 100644 --- a/cmd/client-fs.go +++ b/cmd/client-fs.go @@ -979,6 +979,14 @@ func (f *fsClient) MakeBucket(region string) *probe.Error { return nil } +// GetAccessRules - unsupported API +func (f *fsClient) GetAccessRules() (map[string]string, *probe.Error) { + return map[string]string{}, probe.NewError(APINotImplemented{ + API: "ListBucketPolicies", + APIType: "filesystem", + }) +} + // GetAccess - get access policy permissions. func (f *fsClient) GetAccess() (access string, err *probe.Error) { // For windows this feature is not implemented. diff --git a/cmd/client-s3.go b/cmd/client-s3.go index 42e6ce7722..aed17684d4 100644 --- a/cmd/client-s3.go +++ b/cmd/client-s3.go @@ -678,6 +678,24 @@ func (c *s3Client) MakeBucket(region string) *probe.Error { return nil } +// GetAccessRules - get configured policies from the server +func (c *s3Client) GetAccessRules() (map[string]string, *probe.Error) { + bucket, object := c.url2BucketAndObject() + if bucket == "" { + return map[string]string{}, probe.NewError(BucketNameEmpty{}) + } + policies := map[string]string{} + policyRules, err := c.api.ListBucketPolicies(bucket, object) + if err != nil { + return nil, probe.NewError(err) + } + // Hide policy data structure at this level + for k, v := range policyRules { + policies[k] = string(v) + } + return policies, nil +} + // GetAccess get access policy permissions. func (c *s3Client) GetAccess() (string, *probe.Error) { bucket, object := c.url2BucketAndObject() diff --git a/cmd/client.go b/cmd/client.go index 0f45ca66e3..2c8a0d2aab 100644 --- a/cmd/client.go +++ b/cmd/client.go @@ -35,6 +35,7 @@ type Client interface { // Access policy operations. GetAccess() (access string, error *probe.Error) + GetAccessRules() (policyRules map[string]string, error *probe.Error) SetAccess(access string) *probe.Error // I/O operations diff --git a/cmd/policy-main.go b/cmd/policy-main.go index b6e81f41ca..dbe0c09e9b 100644 --- a/cmd/policy-main.go +++ b/cmd/policy-main.go @@ -72,6 +72,24 @@ EXAMPLES: `, } +// policyRules contains policy rule +type policyRules struct { + Resource string `json:"resource"` + Allow string `json:"allow"` +} + +// String colorized access message. +func (s policyRules) String() string { + return console.Colorize("Policy", s.Resource+" => "+s.Allow+"") +} + +// JSON jsonified policy message. +func (s policyRules) JSON() string { + policyJSONBytes, e := json.Marshal(s) + fatalIf(probe.NewError(e), "Unable to marshal into JSON.") + return string(policyJSONBytes) +} + // policyMessage is container for policy command on bucket success and failure messages. type policyMessage struct { Operation string `json:"operation"` @@ -110,7 +128,7 @@ func checkPolicySyntax(ctx *cli.Context) { if len(ctx.Args()) > 2 { cli.ShowCommandHelpAndExit(ctx, "policy", 1) // last argument is exit code. } - if len(ctx.Args()) == 2 { + if len(ctx.Args()) == 2 && ctx.Args().Get(0) != "list" { perms := accessPerms(ctx.Args().Get(0)) if !perms.isValidAccessPERM() { fatalIf(errDummy().Trace(), @@ -170,6 +188,15 @@ func doGetAccess(targetURL string) (perms accessPerms, err *probe.Error) { return policy, nil } +// doGetAccessRules do get access rules. +func doGetAccessRules(targetURL string) (r map[string]string, err *probe.Error) { + clnt, err := newClient(targetURL) + if err != nil { + return map[string]string{}, err.Trace(targetURL) + } + return clnt.GetAccessRules() +} + func mainPolicy(ctx *cli.Context) { // Set global flags from context. setGlobalsFromContext(ctx) @@ -180,28 +207,39 @@ func mainPolicy(ctx *cli.Context) { // Additional command speific theme customization. console.SetColor("Policy", color.New(color.FgGreen, color.Bold)) - perms := accessPerms(ctx.Args().First()) - if perms.isValidAccessPERM() { + if ctx.Args().First() == "list" { targetURL := ctx.Args().Last() - err := doSetAccess(targetURL, perms) - // Upon error exit. - fatalIf(err.Trace(targetURL, string(perms)), - "Unable to set policy ‘"+string(perms)+"’ for ‘"+targetURL+"’.") - printMsg(policyMessage{ - Status: "success", - Operation: "set", - Bucket: targetURL, - Perms: perms, - }) + policies, err := doGetAccessRules(targetURL) + if err != nil { + fatalIf(err, "Cannot list policies.") + } + for k, v := range policies { + printMsg(policyRules{Resource: k, Allow: v}) + } } else { - targetURL := ctx.Args().First() - perms, err := doGetAccess(targetURL) - fatalIf(err.Trace(targetURL), "Unable to get policy for ‘"+targetURL+"’.") - printMsg(policyMessage{ - Status: "success", - Operation: "get", - Bucket: targetURL, - Perms: perms, - }) + perms := accessPerms(ctx.Args().First()) + if perms.isValidAccessPERM() { + targetURL := ctx.Args().Last() + err := doSetAccess(targetURL, perms) + // Upon error exit. + fatalIf(err.Trace(targetURL, string(perms)), + "Unable to set policy ‘"+string(perms)+"’ for ‘"+targetURL+"’.") + printMsg(policyMessage{ + Status: "success", + Operation: "set", + Bucket: targetURL, + Perms: perms, + }) + } else { + targetURL := ctx.Args().First() + perms, err := doGetAccess(targetURL) + fatalIf(err.Trace(targetURL), "Unable to get policy for ‘"+targetURL+"’.") + printMsg(policyMessage{ + Status: "success", + Operation: "get", + Bucket: targetURL, + Perms: perms, + }) + } } } diff --git a/vendor/github.com/minio/minio-go/api-error-response.go b/vendor/github.com/minio/minio-go/api-error-response.go index 3bfff44edd..bcfad3761a 100644 --- a/vendor/github.com/minio/minio-go/api-error-response.go +++ b/vendor/github.com/minio/minio-go/api-error-response.go @@ -201,16 +201,6 @@ func ErrInvalidObjectName(message string) error { } } -// ErrInvalidParts - Invalid number of parts. -func ErrInvalidParts(expectedParts, uploadedParts int) error { - msg := fmt.Sprintf("Unexpected number of parts found Want %d, Got %d", expectedParts, uploadedParts) - return ErrorResponse{ - Code: "InvalidParts", - Message: msg, - RequestID: "minio", - } -} - // ErrInvalidObjectPrefix - Invalid object prefix response is // similar to object name response. var ErrInvalidObjectPrefix = ErrInvalidObjectName @@ -233,3 +223,13 @@ func ErrNoSuchBucketPolicy(message string) error { RequestID: "minio", } } + +// ErrAPINotSupported - API not supported response +// The specified API call is not supported +func ErrAPINotSupported(message string) error { + return ErrorResponse{ + Code: "APINotSupported", + Message: message, + RequestID: "minio", + } +} diff --git a/vendor/github.com/minio/minio-go/api-get-object.go b/vendor/github.com/minio/minio-go/api-get-object.go index 2d86ff9bcb..b726a422be 100644 --- a/vendor/github.com/minio/minio-go/api-get-object.go +++ b/vendor/github.com/minio/minio-go/api-get-object.go @@ -36,16 +36,13 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { return nil, err } - // Start the request as soon Get is initiated. - httpReader, objectInfo, err := c.getObject(bucketName, objectName, 0, 0) - if err != nil { - return nil, err - } - + var httpReader io.ReadCloser + var objectInfo ObjectInfo + var err error // Create request channel. - reqCh := make(chan readRequest) + reqCh := make(chan getRequest) // Create response channel. - resCh := make(chan readResponse) + resCh := make(chan getResponse) // Create done channel. doneCh := make(chan struct{}) @@ -61,58 +58,148 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { case <-doneCh: // Close the http response body before returning. // This ends the connection with the server. - httpReader.Close() + if httpReader != nil { + httpReader.Close() + } return - // Request message. + + // Gather incoming request. case req := <-reqCh: - // Offset changes fetch the new object at an Offset. - if req.DidOffsetChange { - if httpReader != nil { - // Close previously opened http reader. - httpReader.Close() + // If this is the first request we may not need to do a getObject request yet. + if req.isFirstReq { + // First request is a Read/ReadAt. + if req.isReadOp { + // Differentiate between wanting the whole object and just a range. + if req.isReadAt { + // If this is a ReadAt request only get the specified range. + // Range is set with respect to the offset and length of the buffer requested. + // Do not set objectInfo from the first readAt request because it will not get + // the whole object. + httpReader, _, err = c.getObject(bucketName, objectName, req.Offset, int64(len(req.Buffer))) + } else { + // First request is a Read request. + httpReader, objectInfo, err = c.getObject(bucketName, objectName, req.Offset, 0) + } + if err != nil { + resCh <- getResponse{ + Error: err, + } + return + } + // Read at least firstReq.Buffer bytes, if not we have + // reached our EOF. + size, err := io.ReadFull(httpReader, req.Buffer) + if err == io.ErrUnexpectedEOF { + // If an EOF happens after reading some but not + // all the bytes ReadFull returns ErrUnexpectedEOF + err = io.EOF + } + // Send back the first response. + resCh <- getResponse{ + objectInfo: objectInfo, + Size: int(size), + Error: err, + didRead: true, + } + } else { + // First request is a Stat or Seek call. + // Only need to run a StatObject until an actual Read or ReadAt request comes through. + objectInfo, err = c.StatObject(bucketName, objectName) + if err != nil { + resCh <- getResponse{ + Error: err, + } + // Exit the go-routine. + return + } + // Send back the first response. + resCh <- getResponse{ + objectInfo: objectInfo, + } } - // Read from offset. - httpReader, _, err = c.getObject(bucketName, objectName, req.Offset, 0) + } else if req.settingObjectInfo { // Request is just to get objectInfo. + objectInfo, err := c.StatObject(bucketName, objectName) if err != nil { - resCh <- readResponse{ + resCh <- getResponse{ Error: err, } + // Exit the goroutine. return } - } + // Send back the objectInfo. + resCh <- getResponse{ + objectInfo: objectInfo, + } + } else { + // Offset changes fetch the new object at an Offset. + // Because the httpReader may not be set by the first + // request if it was a stat or seek it must be checked + // if the object has been read or not to only initialize + // new ones when they haven't been already. + // All readAt requests are new requests. + if req.DidOffsetChange || !req.beenRead { + if httpReader != nil { + // Close previously opened http reader. + httpReader.Close() + } + // If this request is a readAt only get the specified range. + if req.isReadAt { + // Range is set with respect to the offset and length of the buffer requested. + httpReader, _, err = c.getObject(bucketName, objectName, req.Offset, int64(len(req.Buffer))) + } else { + httpReader, objectInfo, err = c.getObject(bucketName, objectName, req.Offset, 0) + } + if err != nil { + resCh <- getResponse{ + Error: err, + } + return + } + } - // Read at least req.Buffer bytes, if not we have - // reached our EOF. - size, err := io.ReadFull(httpReader, req.Buffer) - if err == io.ErrUnexpectedEOF { - // If an EOF happens after reading some but not - // all the bytes ReadFull returns ErrUnexpectedEOF - err = io.EOF - } - // Reply back how much was read. - resCh <- readResponse{ - Size: int(size), - Error: err, + // Read at least req.Buffer bytes, if not we have + // reached our EOF. + size, err := io.ReadFull(httpReader, req.Buffer) + if err == io.ErrUnexpectedEOF { + // If an EOF happens after reading some but not + // all the bytes ReadFull returns ErrUnexpectedEOF + err = io.EOF + } + // Reply back how much was read. + resCh <- getResponse{ + Size: int(size), + Error: err, + didRead: true, + objectInfo: objectInfo, + } } } } }() - // Return the readerAt backed by routine. - return newObject(reqCh, resCh, doneCh, objectInfo), nil -} -// Read response message container to reply back for the request. -type readResponse struct { - Size int - Error error + // Create a newObject through the information sent back by reqCh. + return newObject(reqCh, resCh, doneCh), nil } -// Read request message container to communicate with internal +// get request message container to communicate with internal // go-routine. -type readRequest struct { - Buffer []byte - Offset int64 // readAt offset. - DidOffsetChange bool +type getRequest struct { + Buffer []byte + Offset int64 // readAt offset. + DidOffsetChange bool // Tracks the offset changes for Seek requests. + beenRead bool // Determines if this is the first time an object is being read. + isReadAt bool // Determines if this request is a request to a specific range + isReadOp bool // Determines if this request is a Read or Read/At request. + isFirstReq bool // Determines if this request is the first time an object is being accessed. + settingObjectInfo bool // Determines if this request is to set the objectInfo of an object. +} + +// get response message container to reply back for the request. +type getResponse struct { + Size int + Error error + didRead bool // Lets subsequent calls know whether or not httpReader has been initiated. + objectInfo ObjectInfo // Used for the first request. } // Object represents an open object. It implements Read, ReadAt, @@ -122,8 +209,8 @@ type Object struct { mutex *sync.Mutex // User allocated and defined. - reqCh chan<- readRequest - resCh <-chan readResponse + reqCh chan<- getRequest + resCh <-chan getResponse doneCh chan<- struct{} prevOffset int64 currOffset int64 @@ -132,8 +219,60 @@ type Object struct { // Keeps track of closed call. isClosed bool + // Keeps track of if this is the first call. + isStarted bool + // Previous error saved for future calls. prevErr error + + // Keeps track of if this object has been read yet. + beenRead bool + + // Keeps track of if objectInfo has been set yet. + objectInfoSet bool +} + +// doGetRequest - sends and blocks on the firstReqCh and reqCh of an object. +// Returns back the size of the buffer read, if anything was read, as well +// as any error encountered. For all first requests sent on the object +// it is also responsible for sending back the objectInfo. +func (o *Object) doGetRequest(request getRequest) (getResponse, error) { + o.reqCh <- request + response := <-o.resCh + // This was the first request. + if !o.isStarted { + // The object has been operated on. + o.isStarted = true + } + // Set the objectInfo if the request was not readAt + // and it hasn't been set before. + if !o.objectInfoSet && !request.isReadAt { + o.objectInfo = response.objectInfo + o.objectInfoSet = true + } + // Set beenRead only if it has not been set before. + if !o.beenRead { + o.beenRead = response.didRead + } + // Return any error to the top level. + if response.Error != nil { + return response, response.Error + } + return response, nil +} + +// setOffset - handles the setting of offsets for +// Read/ReadAt/Seek requests. +func (o *Object) setOffset(bytesRead int64) error { + // Update the currentOffset. + o.currOffset += bytesRead + // Save the current offset as previous offset. + o.prevOffset = o.currOffset + + if o.currOffset >= o.objectInfo.Size { + return io.EOF + } + return nil } // Read reads up to len(p) bytes into p. It returns the number of @@ -152,16 +291,17 @@ func (o *Object) Read(b []byte) (n int, err error) { if o.prevErr != nil || o.isClosed { return 0, o.prevErr } - - // If current offset has reached Size limit, return EOF. - if o.currOffset >= o.objectInfo.Size { - return 0, io.EOF + // Create a new request. + readReq := getRequest{ + isReadOp: true, + beenRead: o.beenRead, + Buffer: b, } - // Send current information over control channel to indicate we are ready. - reqMsg := readRequest{} - // Send the pointer to the buffer over the channel. - reqMsg.Buffer = b + // Alert that this is the first request. + if !o.isStarted { + readReq.isFirstReq = true + } // Verify if offset has changed and currOffset is greater than // previous offset. Perhaps due to Seek(). @@ -171,42 +311,32 @@ func (o *Object) Read(b []byte) (n int, err error) { } if offsetChange > 0 { // Fetch the new reader at the current offset again. - reqMsg.Offset = o.currOffset - reqMsg.DidOffsetChange = true + readReq.Offset = o.currOffset + readReq.DidOffsetChange = true } else { // No offset changes no need to fetch new reader, continue // reading. - reqMsg.DidOffsetChange = false - reqMsg.Offset = 0 + readReq.DidOffsetChange = false + readReq.Offset = 0 } - // Send read request over the control channel. - o.reqCh <- reqMsg - - // Get data over the response channel. - dataMsg := <-o.resCh + // Send and receive from the first request. + response, err := o.doGetRequest(readReq) + if err != nil { + // Save the error. + o.prevErr = err + return response.Size, err + } // Bytes read. - bytesRead := int64(dataMsg.Size) - - // Update current offset. - o.currOffset += bytesRead + bytesRead := int64(response.Size) - // Save the current offset as previous offset. - o.prevOffset = o.currOffset - - if dataMsg.Error == nil { - // If currOffset read is equal to objectSize - // We have reached end of file, we return io.EOF. - if o.currOffset >= o.objectInfo.Size { - return dataMsg.Size, io.EOF - } - return dataMsg.Size, nil + // Set the new offset. + err = o.setOffset(bytesRead) + if err != nil { + return response.Size, err } - - // Save any error. - o.prevErr = dataMsg.Error - return dataMsg.Size, dataMsg.Error + return response.Size, nil } // Stat returns the ObjectInfo structure describing object. @@ -222,6 +352,21 @@ func (o *Object) Stat() (ObjectInfo, error) { return ObjectInfo{}, o.prevErr } + // This is the first request. + if !o.isStarted || !o.objectInfoSet { + statReq := getRequest{ + isFirstReq: !o.isStarted, + settingObjectInfo: !o.objectInfoSet, + } + + // Send the request and get the response. + _, err := o.doGetRequest(statReq) + if err != nil { + o.prevErr = err + return ObjectInfo{}, err + } + } + return o.objectInfo, nil } @@ -242,57 +387,55 @@ func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { if o.prevErr != nil || o.isClosed { return 0, o.prevErr } - - // if offset is greater than or equal to object size we return io.EOF. - // If offset is negative then we return io.EOF. - if offset < 0 || offset >= o.objectInfo.Size { - return 0, io.EOF + // Can only compare offsets to size when size has been set. + if o.objectInfoSet { + // If offset is negative than we return io.EOF. + // If offset is greater than or equal to object size we return io.EOF. + if offset >= o.objectInfo.Size || offset < 0 { + return 0, io.EOF + } } - // Send current information over control channel to indicate we - // are ready. - reqMsg := readRequest{} - - // Send the offset and pointer to the buffer over the channel. - reqMsg.Buffer = b - - // For ReadAt offset always changes, minor optimization where - // offset same as currOffset we don't change the offset. - reqMsg.DidOffsetChange = offset != o.currOffset - if reqMsg.DidOffsetChange { - // Set new offset. - reqMsg.Offset = offset - // Save new offset as current offset. - o.currOffset = offset + // Create the new readAt request. + readAtReq := getRequest{ + isReadOp: true, + isReadAt: true, + DidOffsetChange: true, // Offset always changes. + beenRead: o.beenRead, // Set if this is the first request to try and read. + Offset: offset, // Set the offset. + Buffer: b, + } + // Alert that this is the first request. + if !o.isStarted { + readAtReq.isFirstReq = true } - // Send read request over the control channel. - o.reqCh <- reqMsg - - // Get data over the response channel. - dataMsg := <-o.resCh - + // Send and receive from the first request. + response, err := o.doGetRequest(readAtReq) + if err != nil { + // Save the error. + o.prevErr = err + return 0, err + } // Bytes read. - bytesRead := int64(dataMsg.Size) - - // Update current offset. - o.currOffset += bytesRead - - // Save current offset as previous offset before returning. - o.prevOffset = o.currOffset - - if dataMsg.Error == nil { - // If currentOffset is equal to objectSize - // we have reached end of file, we return io.EOF. - if o.currOffset >= o.objectInfo.Size { - return dataMsg.Size, io.EOF + bytesRead := int64(response.Size) + // There is no valid objectInfo yet + // to compare against for EOF. + if !o.objectInfoSet { + // Update the currentOffset. + o.currOffset += bytesRead + // Save the current offset as previous offset. + o.prevOffset = o.currOffset + } else { + // If this was not the first request update + // the offsets and compare against objectInfo + // for EOF. + err = o.setOffset(bytesRead) + if err != nil { + return response.Size, err } - return dataMsg.Size, nil } - - // Save any error. - o.prevErr = dataMsg.Error - return dataMsg.Size, dataMsg.Error + return response.Size, nil } // Seek sets the offset for the next Read or Write to offset, @@ -325,6 +468,23 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { return 0, ErrInvalidArgument(fmt.Sprintf("Negative position not allowed for %d.", whence)) } + // This is the first request. So before anything else + // get the ObjectInfo. + if !o.isStarted || !o.objectInfoSet { + // Create the new Seek request. + seekReq := getRequest{ + isReadOp: false, + Offset: offset, + isFirstReq: true, + } + // Send and receive from the seek request. + _, err := o.doGetRequest(seekReq) + if err != nil { + // Save the error. + o.prevErr = err + return 0, err + } + } // Save current offset as previous offset. o.prevOffset = o.currOffset @@ -391,13 +551,13 @@ func (o *Object) Close() (err error) { } // newObject instantiates a new *minio.Object* -func newObject(reqCh chan<- readRequest, resCh <-chan readResponse, doneCh chan<- struct{}, objectInfo ObjectInfo) *Object { +// ObjectInfo will be set by setObjectInfo +func newObject(reqCh chan<- getRequest, resCh <-chan getResponse, doneCh chan<- struct{}) *Object { return &Object{ - mutex: &sync.Mutex{}, - reqCh: reqCh, - resCh: resCh, - doneCh: doneCh, - objectInfo: objectInfo, + mutex: &sync.Mutex{}, + reqCh: reqCh, + resCh: resCh, + doneCh: doneCh, } } @@ -419,6 +579,7 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( customHeader := make(http.Header) // Set ranges if length and offset are valid. + // See https://tools.ietf.org/html/rfc7233#section-3.1 for reference. if length > 0 && offset >= 0 { customHeader.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) } else if offset > 0 && length == 0 { diff --git a/vendor/github.com/minio/minio-go/api-get-policy.go b/vendor/github.com/minio/minio-go/api-get-policy.go index 656727ad3b..07b1fa4836 100644 --- a/vendor/github.com/minio/minio-go/api-get-policy.go +++ b/vendor/github.com/minio/minio-go/api-get-policy.go @@ -41,6 +41,22 @@ func (c Client) GetBucketPolicy(bucketName, objectPrefix string) (bucketPolicy p return policy.GetPolicy(policyInfo.Statements, bucketName, objectPrefix), nil } +// GetBucketPolicy - get bucket policy rules at a given path. +func (c Client) ListBucketPolicies(bucketName, objectPrefix string) (bucketPolicies map[string]policy.BucketPolicy, err error) { + // Input validation. + if err := isValidBucketName(bucketName); err != nil { + return map[string]policy.BucketPolicy{}, err + } + if err := isValidObjectPrefix(objectPrefix); err != nil { + return map[string]policy.BucketPolicy{}, err + } + policyInfo, err := c.getBucketPolicy(bucketName, objectPrefix) + if err != nil { + return map[string]policy.BucketPolicy{}, err + } + return policy.GetPolicies(policyInfo.Statements, bucketName), nil +} + // Request server for policy. func (c Client) getBucketPolicy(bucketName string, objectPrefix string) (policy.BucketAccessPolicy, error) { // Get resources properly escaped and lined up before diff --git a/vendor/github.com/minio/minio-go/api-notification.go b/vendor/github.com/minio/minio-go/api-notification.go index 1b18eb046d..0a2b48c6eb 100644 --- a/vendor/github.com/minio/minio-go/api-notification.go +++ b/vendor/github.com/minio/minio-go/api-notification.go @@ -134,7 +134,15 @@ func (c Client) ListenBucketNotification(bucketName string, accountArn Arn, done return } - // Continuously run and listen on bucket notification. + // Check ARN partition to verify if listening bucket is supported + if accountArn.Partition != "minio" { + notificationInfoCh <- NotificationInfo{ + Err: ErrAPINotSupported("Listening bucket notification is specific only to `minio` partitions"), + } + return + } + + // Continously run and listen on bucket notification. for { urlValues := make(url.Values) urlValues.Set("notificationARN", accountArn.String()) diff --git a/vendor/github.com/minio/minio-go/api-put-object-file.go b/vendor/github.com/minio/minio-go/api-put-object-file.go index 2662bd63a0..199c906740 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-file.go +++ b/vendor/github.com/minio/minio-go/api-put-object-file.go @@ -151,7 +151,7 @@ func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileRe var totalUploadedSize int64 // Complete multipart upload. - var completeMultipartUpload completeMultipartUpload + var complMultipartUpload completeMultipartUpload // A map of all uploaded parts. var partsInfo = make(map[int]objectPart) @@ -236,21 +236,20 @@ func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileRe } // Loop over uploaded parts to save them in a Parts array before completing the multipart request. - for _, part := range partsInfo { - var complPart completePart - complPart.ETag = part.ETag - complPart.PartNumber = part.PartNumber - completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart) - } - - // Verify if totalPartsCount is not equal to total list of parts. - if totalPartsCount != len(completeMultipartUpload.Parts) { - return totalUploadedSize, ErrInvalidParts(partNumber, len(completeMultipartUpload.Parts)) + for i := 1; i <= totalPartsCount; i++ { + part, ok := partsInfo[i] + if !ok { + return totalUploadedSize, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + } + complMultipartUpload.Parts = append(complMultipartUpload.Parts, completePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + }) } // Sort all completed parts. - sort.Sort(completedParts(completeMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload) + sort.Sort(completedParts(complMultipartUpload.Parts)) + _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload) if err != nil { return totalUploadedSize, err } diff --git a/vendor/github.com/minio/minio-go/api-put-object-multipart.go b/vendor/github.com/minio/minio-go/api-put-object-multipart.go index c8332d81f6..463185b080 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-multipart.go +++ b/vendor/github.com/minio/minio-go/api-put-object-multipart.go @@ -22,6 +22,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/xml" + "fmt" "hash" "io" "io/ioutil" @@ -115,7 +116,6 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i tmpBuffer := new(bytes.Buffer) for partNumber <= totalPartsCount { - // Choose hash algorithms to be calculated by hashCopyN, avoid sha256 // with non-v4 signature request or HTTPS connection hashSums := make(map[string][]byte) @@ -187,18 +187,15 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i } // Loop over uploaded parts to save them in a Parts array before completing the multipart request. - for _, part := range partsInfo { - var complPart completePart - complPart.ETag = part.ETag - complPart.PartNumber = part.PartNumber - complMultipartUpload.Parts = append(complMultipartUpload.Parts, complPart) - } - - if size > 0 { - // Verify if totalPartsCount is not equal to total list of parts. - if totalPartsCount != len(complMultipartUpload.Parts) { - return totalUploadedSize, ErrInvalidParts(partNumber, len(complMultipartUpload.Parts)) + for i := 1; i <= totalPartsCount; i++ { + part, ok := partsInfo[i] + if !ok { + return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } + complMultipartUpload.Parts = append(complMultipartUpload.Parts, completePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + }) } // Sort all completed parts. diff --git a/vendor/github.com/minio/minio-go/api-put-object-readat.go b/vendor/github.com/minio/minio-go/api-put-object-readat.go index cd607d5a7d..90c7f9de7e 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-readat.go +++ b/vendor/github.com/minio/minio-go/api-put-object-readat.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto/md5" "crypto/sha256" + "fmt" "hash" "io" "io/ioutil" @@ -187,12 +188,16 @@ func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, read } // Loop over uploaded parts to save them in a Parts array before completing the multipart request. - for _, part := range partsInfo { - var complPart completePart - complPart.ETag = part.ETag - complPart.PartNumber = part.PartNumber + for i := 1; i <= totalPartsCount; i++ { + part, ok := partsInfo[i] + if !ok { + return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + } totalUploadedSize += part.Size - complMultipartUpload.Parts = append(complMultipartUpload.Parts, complPart) + complMultipartUpload.Parts = append(complMultipartUpload.Parts, completePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + }) } // Verify if we uploaded all the data. @@ -200,11 +205,6 @@ func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, read return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) } - // Verify if totalPartsCount is not equal to total list of parts. - if totalPartsCount != len(complMultipartUpload.Parts) { - return totalUploadedSize, ErrInvalidParts(totalPartsCount, len(complMultipartUpload.Parts)) - } - // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload) diff --git a/vendor/github.com/minio/minio-go/api-put-object.go b/vendor/github.com/minio/minio-go/api-put-object.go index ba846f92cb..f7dd2daf1a 100644 --- a/vendor/github.com/minio/minio-go/api-put-object.go +++ b/vendor/github.com/minio/minio-go/api-put-object.go @@ -221,6 +221,9 @@ func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, } defer tmpFile.Close() size, err = hashCopyN(hashAlgos, hashSums, tmpFile, reader, size) + if err != nil { + return 0, err + } // Seek back to beginning of the temporary file. if _, err = tmpFile.Seek(0, 0); err != nil { return 0, err diff --git a/vendor/github.com/minio/minio-go/bucket-notification.go b/vendor/github.com/minio/minio-go/bucket-notification.go index ee76ceb69b..121a63a77a 100644 --- a/vendor/github.com/minio/minio-go/bucket-notification.go +++ b/vendor/github.com/minio/minio-go/bucket-notification.go @@ -18,6 +18,7 @@ package minio import ( "encoding/xml" + "reflect" ) // NotificationEventType is a S3 notification event associated to the bucket notification configuration @@ -160,18 +161,36 @@ type BucketNotification struct { // AddTopic adds a given topic config to the general bucket notification config func (b *BucketNotification) AddTopic(topicConfig NotificationConfig) { newTopicConfig := TopicConfig{NotificationConfig: topicConfig, Topic: topicConfig.Arn.String()} + for _, n := range b.TopicConfigs { + if reflect.DeepEqual(n, newTopicConfig) { + // Avoid adding duplicated entry + return + } + } b.TopicConfigs = append(b.TopicConfigs, newTopicConfig) } // AddQueue adds a given queue config to the general bucket notification config func (b *BucketNotification) AddQueue(queueConfig NotificationConfig) { newQueueConfig := QueueConfig{NotificationConfig: queueConfig, Queue: queueConfig.Arn.String()} + for _, n := range b.QueueConfigs { + if reflect.DeepEqual(n, newQueueConfig) { + // Avoid adding duplicated entry + return + } + } b.QueueConfigs = append(b.QueueConfigs, newQueueConfig) } // AddLambda adds a given lambda config to the general bucket notification config func (b *BucketNotification) AddLambda(lambdaConfig NotificationConfig) { newLambdaConfig := LambdaConfig{NotificationConfig: lambdaConfig, Lambda: lambdaConfig.Arn.String()} + for _, n := range b.LambdaConfigs { + if reflect.DeepEqual(n, newLambdaConfig) { + // Avoid adding duplicated entry + return + } + } b.LambdaConfigs = append(b.LambdaConfigs, newLambdaConfig) } diff --git a/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go b/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go index 067f9d63db..f618059cf5 100644 --- a/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go +++ b/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go @@ -563,6 +563,33 @@ func GetPolicy(statements []Statement, bucketName string, prefix string) BucketP return policy } +// GetPolicies returns a map of policies rules of given bucket name, prefix in given statements. +func GetPolicies(statements []Statement, bucketName string) map[string]BucketPolicy { + policyRules := map[string]BucketPolicy{} + objResources := set.NewStringSet() + // Search all resources related to objects policy + for _, s := range statements { + for r := range s.Resources { + if strings.HasPrefix(r, awsResourcePrefix+bucketName+"/") { + objResources.Add(r) + } + } + } + // Pretend that policy resource as an actual object and fetch its policy + for r := range objResources { + // Put trailing * if exists in asterisk + asterisk := "" + if strings.HasSuffix(r, "*") { + r = r[:len(r)-1] + asterisk = "*" + } + objectPath := r[len(awsResourcePrefix+bucketName)+1 : len(r)] + p := GetPolicy(statements, bucketName, objectPath) + policyRules[bucketName+"/"+objectPath+asterisk] = p + } + return policyRules +} + // Returns new statements containing policy of given bucket name and // prefix are appended. func SetPolicy(statements []Statement, policy BucketPolicy, bucketName string, prefix string) []Statement { diff --git a/vendor/vendor.json b/vendor/vendor.json index 059267dfe0..875a633a0b 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -49,16 +49,16 @@ "revisionTime": "2015-10-24T22:24:27-07:00" }, { - "checksumSHA1": "xk/h/ltr8Er0d0tEFcwLjKUReBE=", + "checksumSHA1": "OA4WJ2M7K2pLCjxoHwy+9UvTDDo=", "path": "github.com/minio/minio-go", - "revision": "9e734013294ab153b0bdbe182738bcddd46f1947", - "revisionTime": "2016-08-18T00:31:20Z" + "revision": "583c261267bc1022bb3e046c7d01c49d3f56edaa", + "revisionTime": "2016-09-03T08:42:23Z" }, { - "checksumSHA1": "0OZaeJPgMlA2Txn+1yeAIwEpJvM=", + "checksumSHA1": "qTxOBp3GVxCC70ykb7Hxg6UgWwA=", "path": "github.com/minio/minio-go/pkg/policy", - "revision": "9e734013294ab153b0bdbe182738bcddd46f1947", - "revisionTime": "2016-08-18T00:31:20Z" + "revision": "583c261267bc1022bb3e046c7d01c49d3f56edaa", + "revisionTime": "2016-09-03T08:42:23Z" }, { "checksumSHA1": "A8QOw1aWwc+RtjGozY0XeS5varo=",