Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature STG 85/86] Cold Tier #21024

Closed
wants to merge 29 commits into from
Closed

[Feature STG 85/86] Cold Tier #21024

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
7b65a42
Regenerating azblob to stg 85/86
siminsavani-msft May 2, 2023
e449fab
Updating CopyFromURL
siminsavani-msft May 2, 2023
703f8bf
minor change
siminsavani-msft May 2, 2023
1eb380f
minor fixes
siminsavani-msft May 2, 2023
b83f0d7
undo some minor fixes
siminsavani-msft May 2, 2023
a1b9a27
Updating Go code generator
siminsavani-msft May 19, 2023
7faac50
Fixing calls to pipeline
siminsavani-msft May 19, 2023
948b091
Adding custom UnmarshalXML for BlobItem and BlobPrefix
siminsavani-msft May 23, 2023
c61c73e
Updating constructor method for AppendBlobClient
siminsavani-msft May 23, 2023
9b5966c
Updating Client constructors
siminsavani-msft Jun 5, 2023
b1b7920
Merge branch 'main' into regenerate-85/86
siminsavani-msft Jun 5, 2023
4127627
Undoing minor fixes to blob examples
siminsavani-msft Jun 5, 2023
6f94001
Merge branch 'regenerate-85/86' of https://github.com/siminsavani-msf…
siminsavani-msft Jun 5, 2023
35d01b9
Fixing authpolicy
siminsavani-msft Jun 12, 2023
1f78a6f
Updating azcore version
siminsavani-msft Jun 12, 2023
d1b507f
Fixing client strings
siminsavani-msft Jun 12, 2023
b1538dd
Const for service client
siminsavani-msft Jun 12, 2023
b6969f5
Merge branch 'main' into regenerate-85/86
siminsavani-msft Jun 12, 2023
059f99a
Minor fix
siminsavani-msft Jun 12, 2023
2338566
fixing go mod files
siminsavani-msft Jun 12, 2023
20775e9
Shared constants client name
siminsavani-msft Jun 13, 2023
0042d66
Adding Cold Tier + tests
siminsavani-msft Jun 15, 2023
0fcdf9f
Merge branch 'feature/azblob-stg85/86' into cold-tier
siminsavani-msft Jun 15, 2023
8fbd3b5
Recorded tests
siminsavani-msft Jun 19, 2023
e603d26
Merge branch 'cold-tier' of https://github.com/siminsavani-msft/azure…
siminsavani-msft Jun 19, 2023
267be2f
Updated CHANGELOG.md
siminsavani-msft Jun 19, 2023
6c7558b
Added recording
siminsavani-msft Jun 19, 2023
ab7f832
Fixing linting issues
siminsavani-msft Jun 19, 2023
c1526fb
Fixing CI issues with test
siminsavani-msft Jun 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions sdk/storage/azblob/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## 1.1.0-beta.2 (Unreleased)

### Features Added
* Added `AccessTierCold`.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added support for Cold tier.


### Breaking Changes

Expand Down
2 changes: 1 addition & 1 deletion sdk/storage/azblob/assets.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "go",
"TagPrefix": "go/storage/azblob",
"Tag": "go/storage/azblob_a772b9c866"
"Tag": "go/storage/azblob_5cbc6d263b"
}
147 changes: 58 additions & 89 deletions sdk/storage/azblob/blob/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3274,95 +3274,64 @@ func (s *BlobRecordedTestsSuite) TestPermanentDeleteWithoutPermission() {
return nil
}*/

//
////func (s *BlobRecordedTestsSuite) TestBlobTierInferred() {
//// svcClient, err := getPremiumserviceClient()
//// if err != nil {
//// c.Skip(err.Error())
//// }
////
//// containerClient, _ := testcommon.CreateNewContainer(c, svcClient)
//// defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
//// bbClient, _ := createNewPageBlob(c, containerClient)
////
//// resp, err := bbClient.GetProperties(context.Background(), nil)
//// _require.Nil(err)
//// _assert(resp.AccessTierInferred(), chk.Equals, "true")
////
//// resp2, err := containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
//// _require.Nil(err)
//// _assert(resp2.Segment.BlobItems[0].Properties.AccessTierInferred, chk.NotNil)
//// _assert(resp2.Segment.BlobItems[0].Properties.AccessTier, chk.Not(chk.Equals), "")
////
//// _, err = bbClient.SetTier(ctx, AccessTierP4, LeaseAccessConditions{})
//// _require.Nil(err)
////
//// resp, err = bbClient.GetProperties(context.Background(), nil)
//// _require.Nil(err)
//// _assert(resp.AccessTierInferred(), chk.Equals, "")
////
//// resp2, err = containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
//// _require.Nil(err)
//// _assert(resp2.Segment.BlobItems[0].Properties.AccessTierInferred, chk.IsNil) // AccessTierInferred never returned if false
////}
////
////func (s *BlobRecordedTestsSuite) TestBlobArchiveStatus() {
//// svcClient, err := getBlobStorageserviceClient()
//// if err != nil {
//// c.Skip(err.Error())
//// }
////
//// containerClient, _ := testcommon.CreateNewContainer(c, svcClient)
//// defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
//// bbClient, _ := createNewBlockBlob(c, containerClient)
////
//// _, err = bbClient.SetTier(ctx, AccessTierArchive, LeaseAccessConditions{})
//// _require.Nil(err)
//// _, err = bbClient.SetTier(ctx, AccessTierCool, LeaseAccessConditions{})
//// _require.Nil(err)
////
//// resp, err := bbClient.GetProperties(context.Background(), nil)
//// _require.Nil(err)
//// _assert(resp.ArchiveStatus(), chk.Equals, string(ArchiveStatusRehydratePendingToCool))
////
//// resp2, err := containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
//// _require.Nil(err)
//// _assert(resp2.Segment.BlobItems[0].Properties.ArchiveStatus, chk.Equals, ArchiveStatusRehydratePendingToCool)
////
//// // delete first blob
//// _, err = bbClient.Delete(context.Background(), DeleteSnapshotsOptionNone, nil)
//// _require.Nil(err)
////
//// bbClient, _ = createNewBlockBlob(c, containerClient)
////
//// _, err = bbClient.SetTier(ctx, AccessTierArchive, LeaseAccessConditions{})
//// _require.Nil(err)
//// _, err = bbClient.SetTier(ctx, AccessTierHot, LeaseAccessConditions{})
//// _require.Nil(err)
////
//// resp, err = bbClient.GetProperties(context.Background(), nil)
//// _require.Nil(err)
//// _assert(resp.ArchiveStatus(), chk.Equals, string(ArchiveStatusRehydratePendingToHot))
////
//// resp2, err = containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
//// _require.Nil(err)
//// _assert(resp2.Segment.BlobItems[0].Properties.ArchiveStatus, chk.Equals, ArchiveStatusRehydratePendingToHot)
////}
////
////func (s *BlobRecordedTestsSuite) TestBlobTierInvalidValue() {
//// svcClient, err := getBlobStorageserviceClient()
//// if err != nil {
//// c.Skip(err.Error())
//// }
////
//// containerClient, _ := testcommon.CreateNewContainer(c, svcClient)
//// defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
//// bbClient, _ := createNewBlockBlob(c, containerClient)
////
//// _, err = bbClient.SetTier(ctx, AccessTierType("garbage"), LeaseAccessConditions{})
//// testcommon.ValidateBlobErrorCode(c, err, bloberror.InvalidHeaderValue)
////}
////
// //func (s *BlobRecordedTestsSuite) TestBlobTierInferred() {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this test commented out?

// // svcClient, err := getPremiumserviceClient()
// // if err != nil {
// // c.Skip(err.Error())
// // }
// //
// // containerClient, _ := testcommon.CreateNewContainer(c, svcClient)
// // defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
// // bbClient, _ := createNewPageBlob(c, containerClient)
// //
// // resp, err := bbClient.GetProperties(context.Background(), nil)
// // _require.Nil(err)
// // _assert(resp.AccessTierInferred(), chk.Equals, "true")
// //
// // resp2, err := containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
// // _require.Nil(err)
// // _assert(resp2.Segment.BlobItems[0].Properties.AccessTierInferred, chk.NotNil)
// // _assert(resp2.Segment.BlobItems[0].Properties.AccessTier, chk.Not(chk.Equals), "")
// //
// // _, err = bbClient.SetTier(ctx, AccessTierP4, LeaseAccessConditions{})
// // _require.Nil(err)
// //
// // resp, err = bbClient.GetProperties(context.Background(), nil)
// // _require.Nil(err)
// // _assert(resp.AccessTierInferred(), chk.Equals, "")
// //
// // resp2, err = containerClient.NewListBlobsFlatPager(ctx, Marker{}, ListBlobsSegmentOptions{})
// // _require.Nil(err)
// // _assert(resp2.Segment.BlobItems[0].Properties.AccessTierInferred, chk.IsNil) // AccessTierInferred never returned if false
// //}
// //

func (s *BlobRecordedTestsSuite) TestBlobSetTierInvalidAndValid() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

blockBlobName := testcommon.GenerateBlobName(testName)
bbClient := testcommon.CreateNewBlockBlob(context.Background(), _require, blockBlobName, containerClient)

_, err = bbClient.SetTier(context.Background(), blob.AccessTier("nothing"), nil)
_require.Error(err)
testcommon.ValidateBlobErrorCode(_require, err, bloberror.InvalidHeaderValue)

for _, tier := range []blob.AccessTier{blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold, blob.AccessTierArchive} {
_, err = bbClient.SetTier(context.Background(), tier, nil)
_require.NoError(err)

getResp, err := bbClient.GetProperties(context.Background(), nil)
_require.NoError(err)
_require.Equal(*getResp.AccessTier, string(tier))
}
}

func (s *BlobRecordedTestsSuite) TestBlobClientPartsSASQueryTimes() {
_require := require.New(s.T())
Expand Down
1 change: 1 addition & 0 deletions sdk/storage/azblob/blob/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ type AccessTier = generated.AccessTier
const (
AccessTierArchive AccessTier = generated.AccessTierArchive
AccessTierCool AccessTier = generated.AccessTierCool
AccessTierCold AccessTier = generated.AccessTierCold
AccessTierHot AccessTier = generated.AccessTierHot
AccessTierP10 AccessTier = generated.AccessTierP10
AccessTierP15 AccessTier = generated.AccessTierP15
Expand Down
88 changes: 85 additions & 3 deletions sdk/storage/azblob/blockblob/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1247,6 +1247,56 @@ func (s *BlockBlobRecordedTestsSuite) TestPutBlobFromURLCopySourceAuthNegative()
testcommon.ValidateBlobErrorCode(_require, err, bloberror.CannotVerifyCopySource)
}

func (s *BlockBlobUnrecordedTestsSuite) TestPutBlobFromURLWithTier() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

src := testcommon.GenerateBlobName("src" + testName)
srcBlob := testcommon.CreateNewBlockBlob(context.Background(), _require, src, containerClient)

// Create SAS for source and get SAS URL
expiryTime := time.Now().UTC().Add(15 * time.Minute)
_require.Nil(err)

credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
_require.NoError(err)

sasQueryParams, err := sas.AccountSignatureValues{
Protocol: sas.ProtocolHTTPS,
ExpiryTime: expiryTime,
Permissions: to.Ptr(sas.AccountPermissions{Read: true, List: true}).String(),
ResourceTypes: to.Ptr(sas.AccountResourceTypes{Container: true, Object: true}).String(),
}.SignWithSharedKey(credential)
_require.Nil(err)

srcBlobParts, _ := blob.ParseURL(srcBlob.URL())
srcBlobParts.SAS = sasQueryParams
srcBlobURLWithSAS := srcBlobParts.String()

for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold} {
dest := testcommon.GenerateBlobName("dest" + string(tier) + testName)
destBlob := testcommon.CreateNewBlockBlob(context.Background(), _require, dest, containerClient)

opts := blockblob.UploadBlobFromURLOptions{
Tier: &tier,
}
// Invoke UploadBlobFromURL
pbResp, err := destBlob.UploadBlobFromURL(context.Background(), srcBlobURLWithSAS, &opts)
_require.NotNil(pbResp)
_require.NoError(err)

getResp, err := destBlob.GetProperties(context.Background(), nil)
_require.Nil(err)
_require.Equal(*getResp.AccessTier, string(tier))
}
}

func (s *BlockBlobRecordedTestsSuite) TestPutBlockListWithImmutabilityPolicy() {
_require := require.New(s.T())
testName := s.T().Name()
Expand Down Expand Up @@ -1909,7 +1959,7 @@ func (s *BlockBlobRecordedTestsSuite) TestSetTierOnBlobUpload() {
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot} {
for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold} {
blobName := strings.ToLower(string(tier)) + testcommon.GenerateBlobName(testName)
bbClient := testcommon.GetBlockBlobClient(blobName, containerClient)

Expand All @@ -1936,7 +1986,7 @@ func (s *BlockBlobRecordedTestsSuite) TestBlobSetTierOnCommit() {
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

for _, tier := range []blob.AccessTier{blob.AccessTierCool, blob.AccessTierHot} {
for _, tier := range []blob.AccessTier{blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold} {
blobName := strings.ToLower(string(tier)) + testcommon.GenerateBlobName(testName)
bbClient := testcommon.GetBlockBlobClient(blobName, containerClient)

Expand All @@ -1955,6 +2005,10 @@ func (s *BlockBlobRecordedTestsSuite) TestBlobSetTierOnCommit() {
_require.NotNil(resp.BlockList.CommittedBlocks)
_require.Nil(resp.BlockList.UncommittedBlocks)
_require.Len(resp.BlockList.CommittedBlocks, 1)

getResp, err := bbClient.GetProperties(context.Background(), nil)
_require.NoError(err)
_require.Equal(*getResp.AccessTier, string(tier))
}
}

Expand Down Expand Up @@ -1998,7 +2052,7 @@ func (s *BlockBlobUnrecordedTestsSuite) TestSetTierOnCopyBlockBlobFromURL() {
srcBlobParts.SAS = sasQueryParams
srcBlobURLWithSAS := srcBlobParts.String()

for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot} {
for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold} {
destBlobName := strings.ToLower(string(tier)) + testcommon.GenerateBlobName(testName)
destBlob := containerClient.NewBlockBlobClient(testcommon.GenerateBlobName(destBlobName))

Expand Down Expand Up @@ -2247,6 +2301,34 @@ func (s *BlockBlobRecordedTestsSuite) TestCopyBlobWithRehydratePriority() {
_require.Equal(*getResp2.ArchiveStatus, string(blob.ArchiveStatusRehydratePendingToHot))
}

func (s *BlockBlobRecordedTestsSuite) TestCopyWithTier() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

for _, tier := range []blob.AccessTier{blob.AccessTierArchive, blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold} {
src := testcommon.GenerateBlobName("src" + string(tier) + testName)
srcBlob := testcommon.CreateNewBlockBlob(context.Background(), _require, src, containerClient)

dest := testcommon.GenerateBlobName("dest" + string(tier) + testName)
destBlob := testcommon.CreateNewBlockBlob(context.Background(), _require, dest, containerClient)

_, err = destBlob.StartCopyFromURL(context.Background(), srcBlob.URL(), &blob.StartCopyFromURLOptions{
Tier: &tier,
})
_require.NoError(err)

getResp, err := destBlob.GetProperties(context.Background(), nil)
_require.Nil(err)
_require.Equal(*getResp.AccessTier, string(tier))
}
}

func (s *BlockBlobRecordedTestsSuite) TestBlobServiceClientDelete() {
_require := require.New(s.T())
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
Expand Down
54 changes: 54 additions & 0 deletions sdk/storage/azblob/container/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1344,6 +1344,60 @@ func (s *ContainerRecordedTestsSuite) TestBlobListWrapper() {
_require.EqualValues(files, found)
}

func (s *ContainerRecordedTestsSuite) TestBlobListColdTier() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.GetContainerClient(containerName, svcClient)

_, err = containerClient.Create(context.Background(), nil)
_require.NoError(err)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

for _, tier := range []blob.AccessTier{blob.AccessTierCool, blob.AccessTierHot, blob.AccessTierCold, blob.AccessTierArchive} {
files := []string{"a123", "b234", "c345"}
testcommon.CreateNewBlobsListTier(context.Background(), _require, files, containerClient, &tier)

// List blobs flat
found := make([]string, 0)
pager := containerClient.NewListBlobsFlatPager(nil)
for pager.More() {
resp, err := pager.NextPage(context.Background())
_require.NoError(err)

for _, blob := range resp.Segment.BlobItems {
_require.Equal(blob.Properties.AccessTier, &tier)
found = append(found, *blob.Name)
}
}

sort.Strings(files)
sort.Strings(found)
_require.EqualValues(files, found)

// Try listing blobs with hierarchical listing
found = make([]string, 0)
pg := containerClient.NewListBlobsHierarchyPager("/", nil)
for pg.More() {
resp, err := pg.NextPage(context.Background())
_require.NoError(err)

for _, blob := range resp.Segment.BlobItems {
_require.Equal(blob.Properties.AccessTier, &tier)
found = append(found, *blob.Name)
}
}

sort.Strings(files)
sort.Strings(found)
_require.EqualValues(files, found)

}
}

func (s *ContainerRecordedTestsSuite) TestBlobListWrapperListingError() {
_require := require.New(s.T())
testName := s.T().Name()
Expand Down
8 changes: 8 additions & 0 deletions sdk/storage/azblob/internal/testcommon/clients_auth.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,14 @@ func CreateNewBlobs(ctx context.Context, _require *require.Assertions, blobNames
}
}

func CreateNewBlobsListTier(ctx context.Context, _require *require.Assertions, blobNames []string, containerClient *container.Client, tier *blob.AccessTier) {
for _, blobName := range blobNames {
bbClient := CreateNewBlockBlob(ctx, _require, blobName, containerClient)
_, err := bbClient.SetTier(ctx, *tier, nil)
_require.NoError(err)
}
}

func GetBlockBlobClient(blockBlobName string, containerClient *container.Client) *blockblob.Client {
return containerClient.NewBlockBlobClient(blockBlobName)
}
Expand Down