Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature STG 85/86] Cold Tier #21032

Merged
merged 6 commits into from
Jun 20, 2023
Merged

[Feature STG 85/86] Cold Tier #21032

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f646d45
Adding Cold Tier + tests
siminsavani-msft Jun 15, 2023
7fa5473
Recorded tests
siminsavani-msft Jun 19, 2023
629f85a
Updated CHANGELOG.md
siminsavani-msft Jun 19, 2023
7c2d270
Fixing linting issues
siminsavani-msft Jun 19, 2023
686ea80
Updating Cold tier test and recording
siminsavani-msft Jun 19, 2023
5940252
Addressing commits
siminsavani-msft Jun 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions sdk/storage/azblob/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## 1.1.0-beta.2 (Unreleased)

### Features Added
* Added `AccessTierCold`.
siminsavani-msft marked this conversation as resolved.
Show resolved Hide resolved

### Breaking Changes

Expand Down
2 changes: 1 addition & 1 deletion sdk/storage/azblob/assets.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "go",
"TagPrefix": "go/storage/azblob",
"Tag": "go/storage/azblob_a772b9c866"
"Tag": "go/storage/azblob_5cbc6d263b"
}
147 changes: 58 additions & 89 deletions sdk/storage/azblob/blob/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3274,95 +3274,64 @@ func (s *BlobRecordedTestsSuite) TestPermanentDeleteWithoutPermission() {
return nil
}*/

//
////func (s *BlobRecordedTestsSuite) TestBlobTierInferred() {
//// svcClient, err := getPremiumserviceClient()
//// if err != nil {
//// c.Skip(err.Error())
//// }
////
//// containerClient, _ := testcommon.CreateNewContainer(c, svcClient)
//// defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
//// bbClient, _ := createNewPageBlob(c, containerClient)
////
//// resp, err := bbClient.GetProperties(context.Background(), nil)
//// _require.Nil(err)
//// _assert(resp.AccessTierInferred(), chk.Equals, "true")
////
//// resp2, err := containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
//// _require.Nil(err)
//// _assert(resp2.Segment.BlobItems[0].Properties.AccessTierInferred, chk.NotNil)
//// _assert(resp2.Segment.BlobItems[0].Properties.AccessTier, chk.Not(chk.Equals), "")
////
//// _, err = bbClient.SetTier(ctx, AccessTierP4, LeaseAccessConditions{})
//// _require.Nil(err)
////
//// resp, err = bbClient.GetProperties(context.Background(), nil)
//// _require.Nil(err)
//// _assert(resp.AccessTierInferred(), chk.Equals, "")
////
//// resp2, err = containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
//// _require.Nil(err)
//// _assert(resp2.Segment.BlobItems[0].Properties.AccessTierInferred, chk.IsNil) // AccessTierInferred never returned if false
////}
////
////func (s *BlobRecordedTestsSuite) TestBlobArchiveStatus() {
//// svcClient, err := getBlobStorageserviceClient()
//// if err != nil {
//// c.Skip(err.Error())
//// }
////
//// containerClient, _ := testcommon.CreateNewContainer(c, svcClient)
//// defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
//// bbClient, _ := createNewBlockBlob(c, containerClient)
////
//// _, err = bbClient.SetTier(ctx, AccessTierArchive, LeaseAccessConditions{})
//// _require.Nil(err)
//// _, err = bbClient.SetTier(ctx, AccessTierCool, LeaseAccessConditions{})
//// _require.Nil(err)
////
//// resp, err := bbClient.GetProperties(context.Background(), nil)
//// _require.Nil(err)
//// _assert(resp.ArchiveStatus(), chk.Equals, string(ArchiveStatusRehydratePendingToCool))
////
//// resp2, err := containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
//// _require.Nil(err)
//// _assert(resp2.Segment.BlobItems[0].Properties.ArchiveStatus, chk.Equals, ArchiveStatusRehydratePendingToCool)
////
//// // delete first blob
//// _, err = bbClient.Delete(context.Background(), DeleteSnapshotsOptionNone, nil)
//// _require.Nil(err)
////
//// bbClient, _ = createNewBlockBlob(c, containerClient)
////
//// _, err = bbClient.SetTier(ctx, AccessTierArchive, LeaseAccessConditions{})
//// _require.Nil(err)
//// _, err = bbClient.SetTier(ctx, AccessTierHot, LeaseAccessConditions{})
//// _require.Nil(err)
////
//// resp, err = bbClient.GetProperties(context.Background(), nil)
//// _require.Nil(err)
//// _assert(resp.ArchiveStatus(), chk.Equals, string(ArchiveStatusRehydratePendingToHot))
////
//// resp2, err = containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
//// _require.Nil(err)
//// _assert(resp2.Segment.BlobItems[0].Properties.ArchiveStatus, chk.Equals, ArchiveStatusRehydratePendingToHot)
////}
////
////func (s *BlobRecordedTestsSuite) TestBlobTierInvalidValue() {
//// svcClient, err := getBlobStorageserviceClient()
//// if err != nil {
//// c.Skip(err.Error())
//// }
////
//// containerClient, _ := testcommon.CreateNewContainer(c, svcClient)
//// defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
//// bbClient, _ := createNewBlockBlob(c, containerClient)
////
//// _, err = bbClient.SetTier(ctx, AccessTierType("garbage"), LeaseAccessConditions{})
//// testcommon.ValidateBlobErrorCode(c, err, bloberror.InvalidHeaderValue)
////}
////
// //func (s *BlobRecordedTestsSuite) TestBlobTierInferred() {
siminsavani-msft marked this conversation as resolved.
Show resolved Hide resolved
// // svcClient, err := getPremiumserviceClient()
// // if err != nil {
// // c.Skip(err.Error())
// // }
// //
// // containerClient, _ := testcommon.CreateNewContainer(c, svcClient)
// // defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
// // bbClient, _ := createNewPageBlob(c, containerClient)
// //
// // resp, err := bbClient.GetProperties(context.Background(), nil)
// // _require.Nil(err)
// // _assert(resp.AccessTierInferred(), chk.Equals, "true")
// //
// // resp2, err := containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
// // _require.Nil(err)
// // _assert(resp2.Segment.BlobItems[0].Properties.AccessTierInferred, chk.NotNil)
// // _assert(resp2.Segment.BlobItems[0].Properties.AccessTier, chk.Not(chk.Equals), "")
// //
// // _, err = bbClient.SetTier(ctx, AccessTierP4, LeaseAccessConditions{})
// // _require.Nil(err)
// //
// // resp, err = bbClient.GetProperties(context.Background(), nil)
// // _require.Nil(err)
// // _assert(resp.AccessTierInferred(), chk.Equals, "")
// //
// // resp2, err = containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
// // _require.Nil(err)
// // _assert(resp2.Segment.BlobItems[0].Properties.AccessTierInferred, chk.IsNil) // AccessTierInferred never returned if false
// //}
// //

func (s *BlobRecordedTestsSuite) TestBlobSetTierInvalidAndValid() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

blockBlobName := testcommon.GenerateBlobName(testName)
bbClient := testcommon.CreateNewBlockBlob(context.Background(), _require, blockBlobName, containerClient)

_, err = bbClient.SetTier(context.Background(), blob.AccessTier("nothing"), nil)
_require.Error(err)
testcommon.ValidateBlobErrorCode(_require, err, bloberror.InvalidHeaderValue)

for _, tier := range []blob.AccessTier{blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold, blob.AccessTierArchive} {
_, err = bbClient.SetTier(context.Background(), tier, nil)
_require.NoError(err)

getResp, err := bbClient.GetProperties(context.Background(), nil)
_require.NoError(err)
_require.Equal(*getResp.AccessTier, string(tier))
}
}

func (s *BlobRecordedTestsSuite) TestBlobClientPartsSASQueryTimes() {
_require := require.New(s.T())
Expand Down
1 change: 1 addition & 0 deletions sdk/storage/azblob/blob/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ type AccessTier = generated.AccessTier
const (
AccessTierArchive AccessTier = generated.AccessTierArchive
AccessTierCool AccessTier = generated.AccessTierCool
AccessTierCold AccessTier = generated.AccessTierCold
AccessTierHot AccessTier = generated.AccessTierHot
AccessTierP10 AccessTier = generated.AccessTierP10
AccessTierP15 AccessTier = generated.AccessTierP15
Expand Down
88 changes: 85 additions & 3 deletions sdk/storage/azblob/blockblob/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1247,6 +1247,56 @@ func (s *BlockBlobRecordedTestsSuite) TestPutBlobFromURLCopySourceAuthNegative()
testcommon.ValidateBlobErrorCode(_require, err, bloberror.CannotVerifyCopySource)
}

func (s *BlockBlobUnrecordedTestsSuite) TestPutBlobFromURLWithTier() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

src := testcommon.GenerateBlobName("src" + testName)
srcBlob := testcommon.CreateNewBlockBlob(context.Background(), _require, src, containerClient)

// Create SAS for source and get SAS URL
expiryTime := time.Now().UTC().Add(15 * time.Minute)
_require.Nil(err)

credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
_require.NoError(err)

sasQueryParams, err := sas.AccountSignatureValues{
Protocol: sas.ProtocolHTTPS,
ExpiryTime: expiryTime,
Permissions: to.Ptr(sas.AccountPermissions{Read: true, List: true}).String(),
ResourceTypes: to.Ptr(sas.AccountResourceTypes{Container: true, Object: true}).String(),
}.SignWithSharedKey(credential)
_require.Nil(err)

srcBlobParts, _ := blob.ParseURL(srcBlob.URL())
srcBlobParts.SAS = sasQueryParams
srcBlobURLWithSAS := srcBlobParts.String()

for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold} {
dest := testcommon.GenerateBlobName("dest" + string(tier) + testName)
destBlob := testcommon.CreateNewBlockBlob(context.Background(), _require, dest, containerClient)

opts := blockblob.UploadBlobFromURLOptions{
Tier: &tier,
}
// Invoke UploadBlobFromURL
pbResp, err := destBlob.UploadBlobFromURL(context.Background(), srcBlobURLWithSAS, &opts)
_require.NotNil(pbResp)
_require.NoError(err)

getResp, err := destBlob.GetProperties(context.Background(), nil)
_require.Nil(err)
_require.Equal(*getResp.AccessTier, string(tier))
}
}

func (s *BlockBlobRecordedTestsSuite) TestPutBlockListWithImmutabilityPolicy() {
_require := require.New(s.T())
testName := s.T().Name()
Expand Down Expand Up @@ -1909,7 +1959,7 @@ func (s *BlockBlobRecordedTestsSuite) TestSetTierOnBlobUpload() {
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot} {
for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold} {
blobName := strings.ToLower(string(tier)) + testcommon.GenerateBlobName(testName)
bbClient := testcommon.GetBlockBlobClient(blobName, containerClient)

Expand All @@ -1936,7 +1986,7 @@ func (s *BlockBlobRecordedTestsSuite) TestBlobSetTierOnCommit() {
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

for _, tier := range []blob.AccessTier{blob.AccessTierCool, blob.AccessTierHot} {
for _, tier := range []blob.AccessTier{blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold} {
blobName := strings.ToLower(string(tier)) + testcommon.GenerateBlobName(testName)
bbClient := testcommon.GetBlockBlobClient(blobName, containerClient)

Expand All @@ -1955,6 +2005,10 @@ func (s *BlockBlobRecordedTestsSuite) TestBlobSetTierOnCommit() {
_require.NotNil(resp.BlockList.CommittedBlocks)
_require.Nil(resp.BlockList.UncommittedBlocks)
_require.Len(resp.BlockList.CommittedBlocks, 1)

getResp, err := bbClient.GetProperties(context.Background(), nil)
_require.NoError(err)
_require.Equal(*getResp.AccessTier, string(tier))
}
}

Expand Down Expand Up @@ -1998,7 +2052,7 @@ func (s *BlockBlobUnrecordedTestsSuite) TestSetTierOnCopyBlockBlobFromURL() {
srcBlobParts.SAS = sasQueryParams
srcBlobURLWithSAS := srcBlobParts.String()

for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot} {
for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold} {
destBlobName := strings.ToLower(string(tier)) + testcommon.GenerateBlobName(testName)
destBlob := containerClient.NewBlockBlobClient(testcommon.GenerateBlobName(destBlobName))

Expand Down Expand Up @@ -2247,6 +2301,34 @@ func (s *BlockBlobRecordedTestsSuite) TestCopyBlobWithRehydratePriority() {
_require.Equal(*getResp2.ArchiveStatus, string(blob.ArchiveStatusRehydratePendingToHot))
}

func (s *BlockBlobRecordedTestsSuite) TestCopyWithTier() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold} {
src := testcommon.GenerateBlobName("src" + string(tier) + testName)
srcBlob := testcommon.CreateNewBlockBlob(context.Background(), _require, src, containerClient)

dest := testcommon.GenerateBlobName("dest" + string(tier) + testName)
destBlob := testcommon.CreateNewBlockBlob(context.Background(), _require, dest, containerClient)

_, err = destBlob.StartCopyFromURL(context.Background(), srcBlob.URL(), &blob.StartCopyFromURLOptions{
Tier: &tier,
})
_require.NoError(err)

getResp, err := destBlob.GetProperties(context.Background(), nil)
_require.Nil(err)
_require.Equal(*getResp.AccessTier, string(tier))
}
}

func (s *BlockBlobRecordedTestsSuite) TestBlobServiceClientDelete() {
_require := require.New(s.T())
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
Expand Down
54 changes: 54 additions & 0 deletions sdk/storage/azblob/container/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1344,6 +1344,60 @@ func (s *ContainerRecordedTestsSuite) TestBlobListWrapper() {
_require.EqualValues(files, found)
}

func (s *ContainerRecordedTestsSuite) TestBlobListColdTier() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.GetContainerClient(containerName, svcClient)

_, err = containerClient.Create(context.Background(), nil)
_require.NoError(err)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

for _, tier := range []blob.AccessTier{blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold, blob.AccessTierArchive} {
files := []string{"a123", "b234", "c345"}
testcommon.CreateNewBlobsListTier(context.Background(), _require, files, containerClient, &tier)

// List blobs flat
found := make([]string, 0)
pager := containerClient.NewListBlobsFlatPager(nil)
for pager.More() {
resp, err := pager.NextPage(context.Background())
_require.NoError(err)

for _, blob := range resp.Segment.BlobItems {
_require.Equal(blob.Properties.AccessTier, &tier)
found = append(found, *blob.Name)
}
}

sort.Strings(files)
sort.Strings(found)
_require.EqualValues(files, found)

// Try listing blobs with hierarchical listing
found = make([]string, 0)
pg := containerClient.NewListBlobsHierarchyPager("/", nil)
for pg.More() {
resp, err := pg.NextPage(context.Background())
_require.NoError(err)

for _, blob := range resp.Segment.BlobItems {
_require.Equal(blob.Properties.AccessTier, &tier)
found = append(found, *blob.Name)
}
}

sort.Strings(files)
sort.Strings(found)
_require.EqualValues(files, found)

}
}

func (s *ContainerRecordedTestsSuite) TestBlobListWrapperListingError() {
_require := require.New(s.T())
testName := s.T().Name()
Expand Down
8 changes: 8 additions & 0 deletions sdk/storage/azblob/internal/testcommon/clients_auth.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,14 @@ func CreateNewBlobs(ctx context.Context, _require *require.Assertions, blobNames
}
}

func CreateNewBlobsListTier(ctx context.Context, _require *require.Assertions, blobNames []string, containerClient *container.Client, tier *blob.AccessTier) {
for _, blobName := range blobNames {
bbClient := CreateNewBlockBlob(ctx, _require, blobName, containerClient)
_, err := bbClient.SetTier(ctx, *tier, nil)
_require.NoError(err)
}
}

func GetBlockBlobClient(blockBlobName string, containerClient *container.Client) *blockblob.Client {
return containerClient.NewBlockBlobClient(blockBlobName)
}
Expand Down