diff --git a/storage/bucket.go b/storage/bucket.go index edd8c43a5b87..e69d1e61e28b 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -491,6 +491,13 @@ type BucketAttrs struct { // 7 day retention duration. In order to fully disable soft delete, you need // to set a policy with a RetentionDuration of 0. SoftDeletePolicy *SoftDeletePolicy + + // HierarchicalNamespace contains the bucket's hierarchical namespace + // configuration. Hierarchical namespace enabled buckets can contain + // [cloud.google.com/go/storage/control/apiv2/controlpb.Folder] resources. + // It cannot be modified after bucket creation time. + // UniformBucketLevelAccess must also also be enabled on the bucket. + HierarchicalNamespace *HierarchicalNamespace } // BucketPolicyOnly is an alias for UniformBucketLevelAccess. @@ -792,6 +799,15 @@ type SoftDeletePolicy struct { RetentionDuration time.Duration } +// HierarchicalNamespace contains the bucket's hierarchical namespace +// configuration. Hierarchical namespace enabled buckets can contain +// [cloud.google.com/go/storage/control/apiv2/controlpb.Folder] resources. +type HierarchicalNamespace struct { + // Enabled indicates whether hierarchical namespace features are enabled on + // the bucket. This can only be set at bucket creation time currently. + Enabled bool +} + func newBucket(b *raw.Bucket) (*BucketAttrs, error) { if b == nil { return nil, nil @@ -830,6 +846,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) { CustomPlacementConfig: customPlacementFromRaw(b.CustomPlacementConfig), Autoclass: toAutoclassFromRaw(b.Autoclass), SoftDeletePolicy: toSoftDeletePolicyFromRaw(b.SoftDeletePolicy), + HierarchicalNamespace: toHierarchicalNamespaceFromRaw(b.HierarchicalNamespace), }, nil } @@ -864,6 +881,7 @@ func newBucketFromProto(b *storagepb.Bucket) *BucketAttrs { ProjectNumber: parseProjectNumber(b.GetProject()), // this can return 0 the project resource name is ID based Autoclass: toAutoclassFromProto(b.GetAutoclass()), SoftDeletePolicy: toSoftDeletePolicyFromProto(b.SoftDeletePolicy), + HierarchicalNamespace: toHierarchicalNamespaceFromProto(b.HierarchicalNamespace), } } @@ -920,6 +938,7 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket { CustomPlacementConfig: b.CustomPlacementConfig.toRawCustomPlacement(), Autoclass: b.Autoclass.toRawAutoclass(), SoftDeletePolicy: b.SoftDeletePolicy.toRawSoftDeletePolicy(), + HierarchicalNamespace: b.HierarchicalNamespace.toRawHierarchicalNamespace(), } } @@ -981,6 +1000,7 @@ func (b *BucketAttrs) toProtoBucket() *storagepb.Bucket { CustomPlacementConfig: b.CustomPlacementConfig.toProtoCustomPlacement(), Autoclass: b.Autoclass.toProtoAutoclass(), SoftDeletePolicy: b.SoftDeletePolicy.toProtoSoftDeletePolicy(), + HierarchicalNamespace: b.HierarchicalNamespace.toProtoHierarchicalNamespace(), } } @@ -2145,6 +2165,42 @@ func toSoftDeletePolicyFromProto(p *storagepb.Bucket_SoftDeletePolicy) *SoftDele } } +func (hns *HierarchicalNamespace) toProtoHierarchicalNamespace() *storagepb.Bucket_HierarchicalNamespace { + if hns == nil { + return nil + } + return &storagepb.Bucket_HierarchicalNamespace{ + Enabled: hns.Enabled, + } +} + +func (hns *HierarchicalNamespace) toRawHierarchicalNamespace() *raw.BucketHierarchicalNamespace { + if hns == nil { + return nil + } + return &raw.BucketHierarchicalNamespace{ + Enabled: hns.Enabled, + } +} + +func toHierarchicalNamespaceFromProto(p *storagepb.Bucket_HierarchicalNamespace) *HierarchicalNamespace { + if p == nil { + return nil + } + return &HierarchicalNamespace{ + Enabled: p.Enabled, + } +} + +func toHierarchicalNamespaceFromRaw(r *raw.BucketHierarchicalNamespace) *HierarchicalNamespace { + if r == nil { + return nil + } + return &HierarchicalNamespace{ + Enabled: r.Enabled, + } +} + // Objects returns an iterator over the objects in the bucket that match the // Query q. If q is nil, no filtering is done. Objects will be iterated over // lexicographically by name. diff --git a/storage/bucket_test.go b/storage/bucket_test.go index eb5312c545a0..de9d2f91426a 100644 --- a/storage/bucket_test.go +++ b/storage/bucket_test.go @@ -62,11 +62,12 @@ func TestBucketAttrsToRawBucket(t *testing.T) { ResponseHeaders: []string{"FOO"}, }, }, - Encryption: &BucketEncryption{DefaultKMSKeyName: "key"}, - Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"}, - Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"}, - Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"}, - SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour}, + Encryption: &BucketEncryption{DefaultKMSKeyName: "key"}, + Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"}, + Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"}, + Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"}, + SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour}, + HierarchicalNamespace: &HierarchicalNamespace{Enabled: true}, Lifecycle: Lifecycle{ Rules: []LifecycleRule{{ Action: LifecycleAction{ @@ -167,11 +168,12 @@ func TestBucketAttrsToRawBucket(t *testing.T) { ResponseHeader: []string{"FOO"}, }, }, - Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"}, - Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"}, - Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"}, - Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"}, - SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 60 * 60}, + Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"}, + Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"}, + Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"}, + Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"}, + SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 60 * 60}, + HierarchicalNamespace: &raw.BucketHierarchicalNamespace{Enabled: true}, Lifecycle: &raw.BucketLifecycle{ Rule: []*raw.BucketLifecycleRule{{ Action: &raw.BucketLifecycleRuleAction{ @@ -665,6 +667,7 @@ func TestNewBucket(t *testing.T) { EffectiveTime: "2017-10-23T04:05:06Z", RetentionDurationSeconds: 3600, }, + HierarchicalNamespace: &raw.BucketHierarchicalNamespace{Enabled: true}, } want := &BucketAttrs{ Name: "name", @@ -726,6 +729,7 @@ func TestNewBucket(t *testing.T) { EffectiveTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC), RetentionDuration: time.Hour, }, + HierarchicalNamespace: &HierarchicalNamespace{Enabled: true}, } got, err := newBucket(rb) if err != nil { @@ -785,6 +789,9 @@ func TestNewBucketFromProto(t *testing.T) { RetentionDuration: durationpb.New(3 * time.Hour), EffectiveTime: toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)), }, + HierarchicalNamespace: &storagepb.Bucket_HierarchicalNamespace{ + Enabled: true, + }, Lifecycle: &storagepb.Bucket_Lifecycle{ Rule: []*storagepb.Bucket_Lifecycle_Rule{ { @@ -830,6 +837,9 @@ func TestNewBucketFromProto(t *testing.T) { EffectiveTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), RetentionDuration: time.Hour * 3, }, + HierarchicalNamespace: &HierarchicalNamespace{ + Enabled: true, + }, Lifecycle: Lifecycle{ Rules: []LifecycleRule{{ Action: LifecycleAction{ @@ -874,11 +884,12 @@ func TestBucketAttrsToProtoBucket(t *testing.T) { ResponseHeaders: []string{"FOO"}, }, }, - Encryption: &BucketEncryption{DefaultKMSKeyName: "key"}, - Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"}, - Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"}, - Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"}, - SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour * 2}, + Encryption: &BucketEncryption{DefaultKMSKeyName: "key"}, + Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"}, + Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"}, + Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"}, + SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour * 2}, + HierarchicalNamespace: &HierarchicalNamespace{Enabled: true}, Lifecycle: Lifecycle{ Rules: []LifecycleRule{{ Action: LifecycleAction{ @@ -925,11 +936,12 @@ func TestBucketAttrsToProtoBucket(t *testing.T) { ResponseHeader: []string{"FOO"}, }, }, - Encryption: &storagepb.Bucket_Encryption{DefaultKmsKey: "key"}, - Logging: &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"}, - Website: &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"}, - Autoclass: &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC}, - SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{RetentionDuration: durationpb.New(2 * time.Hour)}, + Encryption: &storagepb.Bucket_Encryption{DefaultKmsKey: "key"}, + Logging: &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"}, + Website: &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"}, + Autoclass: &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC}, + SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{RetentionDuration: durationpb.New(2 * time.Hour)}, + HierarchicalNamespace: &storagepb.Bucket_HierarchicalNamespace{Enabled: true}, Lifecycle: &storagepb.Bucket_Lifecycle{ Rule: []*storagepb.Bucket_Lifecycle_Rule{ { diff --git a/storage/integration_test.go b/storage/integration_test.go index 9764abd535c9..74a06021be6f 100644 --- a/storage/integration_test.go +++ b/storage/integration_test.go @@ -1012,6 +1012,50 @@ func TestIntegration_ConditionalDelete(t *testing.T) { }) } +func TestIntegration_HierarchicalNamespace(t *testing.T) { + ctx := skipJSONReads(context.Background(), "no reads in test") + multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, prefix string, client *Client) { + h := testHelper{t} + + // Create a bucket with HNS enabled. + hnsBucketName := prefix + uidSpace.New() + bkt := client.Bucket(hnsBucketName) + h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ + UniformBucketLevelAccess: UniformBucketLevelAccess{Enabled: true}, + HierarchicalNamespace: &HierarchicalNamespace{Enabled: true}, + }) + defer h.mustDeleteBucket(bkt) + + attrs, err := bkt.Attrs(ctx) + if err != nil { + t.Fatalf("bkt(%q).Attrs: %v", hnsBucketName, err) + } + + if got, want := (attrs.HierarchicalNamespace), (&HierarchicalNamespace{Enabled: true}); cmp.Diff(got, want) != "" { + t.Errorf("HierarchicalNamespace: got %+v, want %+v", got, want) + } + + // Folder creation should work on HNS bucket, but not on standard bucket. + req := &controlpb.CreateFolderRequest{ + Parent: fmt.Sprintf("projects/_/buckets/%v", hnsBucketName), + FolderId: "foo/", + Folder: &controlpb.Folder{}, + } + if _, err := controlClient.CreateFolder(ctx, req); err != nil { + t.Errorf("creating folder in bucket %q: %v", hnsBucketName, err) + } + + req2 := &controlpb.CreateFolderRequest{ + Parent: fmt.Sprintf("projects/_/buckets/%v", bucket), + FolderId: "foo/", + Folder: &controlpb.Folder{}, + } + if _, err := controlClient.CreateFolder(ctx, req2); status.Code(err) != codes.FailedPrecondition { + t.Errorf("creating folder in non-HNS bucket %q: got error %v, want FailedPrecondition", bucket, err) + } + }) +} + func TestIntegration_ObjectsRangeReader(t *testing.T) { multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { bkt := client.Bucket(bucket) @@ -6046,6 +6090,32 @@ func killBucket(ctx context.Context, client *Client, bucketName string) error { } } + // Delete any folders. + listFoldersReq := &controlpb.ListFoldersRequest{ + Parent: fmt.Sprintf("projects/_/buckets/%s", bucketName), + } + folderIt := controlClient.ListFolders(ctx, listFoldersReq) + for { + resp, err := folderIt.Next() + if err == iterator.Done { + break + } + // Buckets without UBLA will return this error for Folder ops; skip. + if status.Code(err) == codes.FailedPrecondition { + break + } + if err != nil { + return err + } + deleteFolderReq := &controlpb.DeleteFolderRequest{ + Name: resp.Name, + } + err = controlClient.DeleteFolder(ctx, deleteFolderReq) + if err != nil { + return err + } + } + // GCS is eventually consistent, so this delete may fail because the // replica still sees an object in the bucket. We log the error and expect // a later test run to delete the bucket.