Add custom MD5/SHA256 hasher option (#1283)

api-put-object-multipart.go

@@ -118,6 +118,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje
 		for k, v := range hashAlgos {
 			v.Write(buf[:length])
 			hashSums[k] = v.Sum(nil)
+			v.Close()
 		}
 
 		// Update progress reader appropriately to the latest offset

api-put-object-streaming.go

@@ -20,7 +20,6 @@ package minio
 import (
 	"bytes"
 	"context"
-	"crypto/md5"
 	"encoding/base64"
 	"fmt"
 	"io"
@@ -287,9 +286,11 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu
 				return 0, rerr
 			}
 			// Calculate md5sum.
-			hash := md5.New()
+			hash := c.md5Hasher()
 			hash.Write(buf[:length])
 			md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil))
+			hash.Close()
+
 			// Update progress reader appropriately to the latest offset
 			// as we read from the source.
 			hookReader = newHook(bytes.NewReader(buf[:length]), opts.Progress)
@@ -393,10 +394,11 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re
 		}
 
 		// Calculate md5sum.
-		hash := md5.New()
+		hash := c.md5Hasher()
 		hash.Write(buf[:length])
 		md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil))
 		reader = bytes.NewReader(buf[:length])
+		hash.Close()
 	}
 
 	// Update progress reader appropriately to the latest offset as we

api-put-object.go

@@ -20,7 +20,6 @@ package minio
 import (
 	"bytes"
 	"context"
-	"crypto/md5"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -267,9 +266,10 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName
 		var md5Base64 string
 		if opts.SendContentMd5 {
 			// Calculate md5sum.
-			hash := md5.New()
+			hash := c.md5Hasher()
 			hash.Write(buf[:length])
 			md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil))
+			hash.Close()
 		}
 
 		// Update progress reader appropriately to the latest offset

api.go

@@ -20,10 +20,8 @@ package minio
 import (
 	"bytes"
 	"context"
-	"crypto/md5"
 	"errors"
 	"fmt"
-	"hash"
 	"io"
 	"io/ioutil"
 	"math/rand"
@@ -38,13 +36,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/sha256-simd"
-
-	"golang.org/x/net/publicsuffix"
-
+	md5simd "github.com/minio/md5-simd"
 	"github.com/minio/minio-go/v6/pkg/credentials"
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio-go/v6/pkg/signer"
+	"golang.org/x/net/publicsuffix"
 )
 
 // Client implements Amazon S3 compatible methods.
@@ -90,6 +86,10 @@ type Client struct {
 	// lookup indicates type of url lookup supported by server. If not specified,
 	// default to Auto.
 	lookup BucketLookupType
+
+	// Factory for MD5 hash functions.
+	md5Hasher    func() md5simd.Hasher
+	sha256Hasher func() md5simd.Hasher
 }
 
 // Options for New method
@@ -98,7 +98,10 @@ type Options struct {
 	Secure       bool
 	Region       string
 	BucketLookup BucketLookupType
-	// Add future fields here
+
+	// Custom hash routines. Leave nil to use standard.
+	CustomMD5    func() md5simd.Hasher
+	CustomSHA256 func() md5simd.Hasher
 }
 
 // Global constants.
@@ -129,8 +132,11 @@ const (
 // NewV2 - instantiate minio client with Amazon S3 signature version
 // '2' compatibility.
 func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) {
-	creds := credentials.NewStaticV2(accessKeyID, secretAccessKey, "")
-	clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto)
+	clnt, err := privateNew(endpoint, Options{
+		Creds:        credentials.NewStaticV2(accessKeyID, secretAccessKey, ""),
+		Secure:       secure,
+		BucketLookup: BucketLookupAuto,
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -141,8 +147,11 @@ func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*
 // NewV4 - instantiate minio client with Amazon S3 signature version
 // '4' compatibility.
 func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) {
-	creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "")
-	clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto)
+	clnt, err := privateNew(endpoint, Options{
+		Creds:        credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure:       secure,
+		BucketLookup: BucketLookupAuto,
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -152,8 +161,11 @@ func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*
 
 // New - instantiate minio client, adds automatic verification of signature.
 func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) {
-	creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "")
-	clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto)
+	clnt, err := privateNew(endpoint, Options{
+		Creds:        credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure:       secure,
+		BucketLookup: BucketLookupAuto,
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -172,20 +184,33 @@ func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, e
 // for retrieving credentials from various credentials provider such as
 // IAM, File, Env etc.
 func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) {
-	return privateNew(endpoint, creds, secure, region, BucketLookupAuto)
+	return privateNew(endpoint, Options{
+		Creds:        creds,
+		Secure:       secure,
+		BucketLookup: BucketLookupAuto,
+		Region:       region,
+	})
 }
 
 // NewWithRegion - instantiate minio client, with region configured. Unlike New(),
 // NewWithRegion avoids bucket-location lookup operations and it is slightly faster.
 // Use this function when if your application deals with single region.
 func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) {
 	creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "")
-	return privateNew(endpoint, creds, secure, region, BucketLookupAuto)
+	return privateNew(endpoint, Options{
+		Creds:        creds,
+		Secure:       secure,
+		Region:       region,
+		BucketLookup: BucketLookupAuto,
+	})
 }
 
 // NewWithOptions - instantiate minio client with options
 func NewWithOptions(endpoint string, opts *Options) (*Client, error) {
-	return privateNew(endpoint, opts.Creds, opts.Secure, opts.Region, opts.BucketLookup)
+	if opts == nil {
+		return nil, errors.New("no options provided")
+	}
+	return privateNew(endpoint, *opts)
 }
 
 // EndpointURL returns the URL of the S3 endpoint.
@@ -277,9 +302,9 @@ func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error {
 	return nil
 }
 
-func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string, lookup BucketLookupType) (*Client, error) {
+func privateNew(endpoint string, opts Options) (*Client, error) {
 	// construct endpoint.
-	endpointURL, err := getEndpointURL(endpoint, secure)
+	endpointURL, err := getEndpointURL(endpoint, opts.Secure)
 	if err != nil {
 		return nil, err
 	}
@@ -295,15 +320,15 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re
 	clnt := new(Client)
 
 	// Save the credentials.
-	clnt.credsProvider = creds
+	clnt.credsProvider = opts.Creds
 
 	// Remember whether we are using https or not
-	clnt.secure = secure
+	clnt.secure = opts.Secure
 
 	// Save endpoint URL, user agent for future uses.
 	clnt.endpointURL = endpointURL
 
-	transport, err := DefaultTransport(secure)
+	transport, err := DefaultTransport(opts.Secure)
 	if err != nil {
 		return nil, err
 	}
@@ -316,20 +341,29 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re
 	}
 
 	// Sets custom region, if region is empty bucket location cache is used automatically.
-	if region == "" {
-		region = s3utils.GetRegionFromURL(*clnt.endpointURL)
+	if opts.Region == "" {
+		opts.Region = s3utils.GetRegionFromURL(*clnt.endpointURL)
 	}
-	clnt.region = region
+	clnt.region = opts.Region
 
 	// Instantiate bucket location cache.
 	clnt.bucketLocCache = newBucketLocationCache()
 
 	// Introduce a new locked random seed.
 	clnt.random = rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())})
 
+	// Add default md5 hasher.
+	clnt.md5Hasher = opts.CustomMD5
+	clnt.sha256Hasher = opts.CustomSHA256
+	if clnt.md5Hasher == nil {
+		clnt.md5Hasher = newMd5Hasher
+	}
+	if clnt.sha256Hasher == nil {
+		clnt.sha256Hasher = newSHA256Hasher
+	}
 	// Sets bucket lookup style, whether server accepts DNS or Path lookup. Default is Auto - determined
 	// by the SDK. When Auto is specified, DNS lookup is used for Amazon/Google cloud endpoints and Path for all other endpoints.
-	clnt.lookup = lookup
+	clnt.lookup = opts.BucketLookup
 	// Return.
 	return clnt, nil
 }
@@ -413,22 +447,22 @@ func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) {
 //  - For signature v4 request if the connection is insecure compute only sha256.
 //  - For signature v4 request if the connection is secure compute only md5.
 //  - For anonymous request compute md5.
-func (c *Client) hashMaterials(isMd5Requested bool) (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) {
+func (c *Client) hashMaterials(isMd5Requested bool) (hashAlgos map[string]md5simd.Hasher, hashSums map[string][]byte) {
 	hashSums = make(map[string][]byte)
-	hashAlgos = make(map[string]hash.Hash)
+	hashAlgos = make(map[string]md5simd.Hasher)
 	if c.overrideSignerType.IsV4() {
 		if c.secure {
-			hashAlgos["md5"] = md5.New()
+			hashAlgos["md5"] = c.md5Hasher()
 		} else {
-			hashAlgos["sha256"] = sha256.New()
+			hashAlgos["sha256"] = c.sha256Hasher()
 		}
 	} else {
 		if c.overrideSignerType.IsAnonymous() {
-			hashAlgos["md5"] = md5.New()
+			hashAlgos["md5"] = c.md5Hasher()
 		}
 	}
 	if isMd5Requested {
-		hashAlgos["md5"] = md5.New()
+		hashAlgos["md5"] = c.md5Hasher()
 	}
 	return hashAlgos, hashSums
 }

core_test.go

@@ -678,7 +678,8 @@ func TestCorePutObject(t *testing.T) {
 		mustParseBool(os.Getenv(enableSecurity)),
 	)
 	if err != nil {
-		t.Fatal("Error:", err)
+		t.Error("Error:", err)
+		return
 	}
 
 	// Enable tracing, write to stderr.

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.9
+	github.com/minio/md5-simd v1.1.0
 	github.com/minio/sha256-simd v0.1.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/sirupsen/logrus v1.5.0 // indirect

go.sum

@@ -10,7 +10,13 @@ github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGn
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs=
+github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/minio/md5-simd v1.0.1 h1:tj/FH8APTKxIkOGUX2YGAVJVXXC3AJ5T2SkHoT/dUFI=
+github.com/minio/md5-simd v1.0.1/go.mod h1:EhdyA+Dr0guvfyc8d6yrgs9YzHGfaI+YIjA6gt/7mJk=
+github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
+github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -21,6 +27,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
 github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q=
 github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=