[Checksum Work] BlockBlob Client: Transactional/Source Content Valida…

…tion (#21033)

* Adding test for transactional validation in block blob

* StageBlockFromURL tests

* Deprecating options in CommitBlockList

* CopyFromURL test

* Updating checksum behavior

* Record tests

* Updated recording

* Updated recording again

* Added error for user gen checksums, tests for UploadStream, UploadBuffer, and UploadFile

* Added recorded test

* Updated CommitBlockList, added tests for CommitBlockList, and added CRC64 test for Upload

* Updating UploadStream test

* Recorded test

* Recorded test

* Fixing CommitBlockList errors

* Fixing linting issues

* Addressing comment + handling CI issues

* Removing TransactionalValidation from CommitBlockList + cleaning up tests

* Rerecorded tests

sdk/storage/azblob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "go",
   "TagPrefix": "go/storage/azblob",
-  "Tag": "go/storage/azblob_1a8d8f30f5"
+  "Tag": "go/storage/azblob_23a06ae998"
 }

sdk/storage/azblob/blob/client_test.go

@@ -191,67 +191,6 @@ func waitForCopy(_require *require.Assertions, copyBlobClient *blockblob.Client,
 	}
 }
 
-func (s *BlobUnrecordedTestsSuite) TestCopyBlockBlobFromUrlSourceContentMD5() {
-	_require := require.New(s.T())
-	testName := s.T().Name()
-	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
-	if err != nil {
-		s.Fail("Unable to fetch service client because " + err.Error())
-	}
-
-	containerName := testcommon.GenerateContainerName(testName)
-	containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
-	defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
-
-	const contentSize = 8 * 1024 // 8 KB
-	content := make([]byte, contentSize)
-	contentMD5 := md5.Sum(content)
-	body := bytes.NewReader(content)
-
-	srcBlob := containerClient.NewBlockBlobClient("srcblob")
-	destBlob := containerClient.NewBlockBlobClient("destblob")
-
-	// Prepare source bbClient for copy.
-	_, err = srcBlob.Upload(context.Background(), streaming.NopCloser(body), nil)
-	_require.Nil(err)
-
-	expiryTime, err := time.Parse(time.UnixDate, "Fri Jun 11 20:00:00 UTC 2049")
-	_require.Nil(err)
-
-	credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
-	if err != nil {
-		s.T().Fatal("Couldn't fetch credential because " + err.Error())
-	}
-
-	// Get source blob url with SAS for StageFromURL.
-	sasQueryParams, err := sas.AccountSignatureValues{
-		Protocol:      sas.ProtocolHTTPS,
-		ExpiryTime:    expiryTime,
-		Permissions:   to.Ptr(sas.AccountPermissions{Read: true, List: true}).String(),
-		ResourceTypes: to.Ptr(sas.AccountResourceTypes{Container: true, Object: true}).String(),
-	}.SignWithSharedKey(credential)
-	_require.Nil(err)
-
-	srcBlobParts, _ := blob.ParseURL(srcBlob.URL())
-	srcBlobParts.SAS = sasQueryParams
-	srcBlobURLWithSAS := srcBlobParts.String()
-
-	// Invoke CopyFromURL.
-	sourceContentMD5 := contentMD5[:]
-	resp, err := destBlob.CopyFromURL(context.Background(), srcBlobURLWithSAS, &blob.CopyFromURLOptions{
-		SourceContentMD5: sourceContentMD5,
-	})
-	_require.Nil(err)
-	_require.EqualValues(resp.ContentMD5, sourceContentMD5)
-
-	// Provide bad MD5 and make sure the copy fails
-	_, badMD5 := testcommon.GetDataAndReader(testName, 16)
-	resp, err = destBlob.CopyFromURL(context.Background(), srcBlobURLWithSAS, &blob.CopyFromURLOptions{
-		SourceContentMD5: badMD5,
-	})
-	_require.NotNil(err)
-}
-
 func (s *BlobRecordedTestsSuite) TestBlobStartCopyDestEmpty() {
 	_require := require.New(s.T())
 	testName := s.T().Name()

sdk/storage/azblob/bloberror/error_codes.go

@@ -153,4 +153,5 @@ const (
 var (
 	// MissingSharedKeyCredential - Error is returned when SAS URL is being created without SharedKeyCredential.
 	MissingSharedKeyCredential = errors.New("SAS can only be signed with a SharedKeyCredential")
+	UnsupportedChecksum        = errors.New("for multi-part uploads, user generated checksums cannot be validated")
 )

sdk/storage/azblob/blockblob/client.go

@@ -12,9 +12,11 @@ import (
 	"encoding/base64"
 	"errors"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
 	"io"
 	"math"
 	"os"
+	"reflect"
 	"sync"
 	"time"
 
@@ -172,6 +174,13 @@ func (bb *Client) Upload(ctx context.Context, body io.ReadSeekCloser, options *U
 
 	opts, httpHeaders, leaseInfo, cpkV, cpkN, accessConditions := options.format()
 
+	if options != nil && options.TransactionalValidation != nil {
+		body, err = options.TransactionalValidation.Apply(body, opts)
+		if err != nil {
+			return UploadResponse{}, err
+		}
+	}
+
 	resp, err := bb.generated().Upload(ctx, count, body, opts, httpHeaders, leaseInfo, cpkV, cpkN, accessConditions)
 	return resp, err
 }
@@ -260,6 +269,11 @@ func (bb *Client) CommitBlockList(ctx context.Context, base64BlockIDs []string,
 			ImmutabilityPolicyExpiry:  options.ImmutabilityPolicyExpiryTime,
 		}
 
+		// If user attempts to pass in their own checksum, errors out.
+		if options.TransactionalContentMD5 != nil || options.TransactionalContentCRC64 != nil {
+			return CommitBlockListResponse{}, bloberror.UnsupportedChecksum
+		}
+
 		headers = options.HTTPHeaders
 		leaseAccess, modifiedAccess = exported.FormatBlobAccessConditions(options.AccessConditions)
 		cpkInfo = options.CPKInfo
@@ -506,6 +520,12 @@ func (bb *Client) UploadBuffer(ctx context.Context, buffer []byte, o *UploadBuff
 	if o != nil {
 		uploadOptions = *o
 	}
+
+	// If user attempts to pass in their own checksum, errors out.
+	if uploadOptions.TransactionalValidation != nil && reflect.TypeOf(uploadOptions.TransactionalValidation).Kind() != reflect.Func {
+		return UploadBufferResponse{}, bloberror.UnsupportedChecksum
+	}
+
 	return bb.uploadFromReader(ctx, bytes.NewReader(buffer), int64(len(buffer)), &uploadOptions)
 }
 
@@ -519,6 +539,12 @@ func (bb *Client) UploadFile(ctx context.Context, file *os.File, o *UploadFileOp
 	if o != nil {
 		uploadOptions = *o
 	}
+
+	// If user attempts to pass in their own checksum, errors out.
+	if uploadOptions.TransactionalValidation != nil && reflect.TypeOf(uploadOptions.TransactionalValidation).Kind() != reflect.Func {
+		return UploadFileResponse{}, bloberror.UnsupportedChecksum
+	}
+
 	return bb.uploadFromReader(ctx, file, stat.Size(), &uploadOptions)
 }
 
@@ -529,6 +555,11 @@ func (bb *Client) UploadStream(ctx context.Context, body io.Reader, o *UploadStr
 		o = &UploadStreamOptions{}
 	}
 
+	// If user attempts to pass in their own checksum, errors out.
+	if o.TransactionalValidation != nil && reflect.TypeOf(o.TransactionalValidation).Kind() != reflect.Func {
+		return UploadStreamResponse{}, bloberror.UnsupportedChecksum
+	}
+
 	result, err := copyFromReader(ctx, body, bb, *o, newMMBPool)
 	if err != nil {
 		return CommitBlockListResponse{}, err