Skip to content

Commit

Permalink
storage - back-port storage data plane workarounds (#27819)
Browse files Browse the repository at this point in the history
* replace on branched fixes for base layer to allow testing

* squash of experimental commits to clean up

* make generate

* add changelog for manual release

* changes to match review on related PR

* review feedback

* update changelog in readiness for manual release

* restore return read

* prep changelog for manual release
  • Loading branch information
jackofallops authored Nov 5, 2024
1 parent c3672e9 commit 7e34ba7
Show file tree
Hide file tree
Showing 35 changed files with 2,315 additions and 132 deletions.
22 changes: 8 additions & 14 deletions .github/labeler-issue-triage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ service/database-migration:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_database_migration_((.|\n)*)###'

service/databox-edge:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_databox_edge_((.|\n)*)###'
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_databox_edge_device((.|\n)*)###'

service/databricks:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_databricks_((.|\n)*)###'
Expand All @@ -121,7 +121,6 @@ service/digital-twins:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_digital_twins_((.|\n)*)###'

service/disks:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_disk_pool((.|\n)*)###'

service/dns:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(dns_|private_dns_a_record\W+|private_dns_aaaa_record\W+|private_dns_cname_record\W+|private_dns_mx_record\W+|private_dns_ptr_record\W+|private_dns_soa_record\W+|private_dns_srv_record\W+|private_dns_txt_record\W+|private_dns_zone\W+|private_dns_zone_virtual_network_link\W+)((.|\n)*)###'
Expand Down Expand Up @@ -154,7 +153,7 @@ service/frontdoor:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_frontdoor((.|\n)*)###'

service/graph:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_graph_((.|\n)*)###'
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_graph_services_account((.|\n)*)###'

service/hdinsight:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_hdinsight_((.|\n)*)###'
Expand All @@ -166,7 +165,7 @@ service/hsm:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_dedicated_hardware_security_module((.|\n)*)###'

service/hybrid-compute:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(arc_machine\W+|arc_machine_extension\W+|arc_private_link_scope\W+|hybrid_compute_machine)((.|\n)*)###'
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(arc_machine\W+|arc_machine_extension\W+|arc_private_link_scope\W+)((.|\n)*)###'

service/iot-central:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_iotcentral_((.|\n)*)###'
Expand All @@ -175,7 +174,6 @@ service/iot-hub:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_iothub((.|\n)*)###'

service/iot-time-series:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_iot_time_series_insights_((.|\n)*)###'

service/key-vault:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(key_vault\W+|key_vault_access_policy\W+|key_vault_certificate\W+|key_vault_certificate_contacts\W+|key_vault_certificate_data\W+|key_vault_certificate_issuer\W+|key_vault_certificates\W+|key_vault_encrypted_value\W+|key_vault_key\W+|key_vault_managed_storage_account\W+|key_vault_managed_storage_account_sas_token_definition\W+|key_vault_secret\W+|key_vault_secrets\W+)((.|\n)*)###'
Expand All @@ -184,7 +182,6 @@ service/kusto:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_kusto_((.|\n)*)###'

service/labservice:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_lab_service_((.|\n)*)###'

service/lighthouse:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_lighthouse_((.|\n)*)###'
Expand All @@ -199,10 +196,9 @@ service/log-analytics:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_log_analytics_((.|\n)*)###'

service/logic:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(integration_service_environment|logic_app_)((.|\n)*)###'
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_logic_app_((.|\n)*)###'

service/logz:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_logz_((.|\n)*)###'

service/machine-learning:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_machine_learning_((.|\n)*)###'
Expand All @@ -226,7 +222,7 @@ service/maria-db:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_mariadb_((.|\n)*)###'

service/media:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_media_((.|\n)*)###'
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_media_services_account_filter((.|\n)*)###'

service/mixed-reality:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_spatial_anchors_account((.|\n)*)###'
Expand All @@ -244,7 +240,7 @@ service/mssqlmanagedinstance:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_mssql_managed_((.|\n)*)###'

service/mysql:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_mysql_((.|\n)*)###'
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_mysql_flexible_((.|\n)*)###'

service/netapp:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_netapp_((.|\n)*)###'
Expand All @@ -271,7 +267,7 @@ service/policy:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(management_group_policy_|policy_|resource_group_policy_assignment\W+|resource_group_policy_exemption\W+|resource_group_policy_remediation\W+|resource_policy_assignment\W+|resource_policy_exemption\W+|resource_policy_remediation\W+|subscription_policy_)((.|\n)*)###'

service/portal:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(dashboard\W+|portal_)((.|\n)*)###'
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_portal_((.|\n)*)###'

service/postgresql:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_postgresql_((.|\n)*)###'
Expand Down Expand Up @@ -328,10 +324,9 @@ service/spring:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(spring_cloud_accelerator\W+|spring_cloud_active_deployment\W+|spring_cloud_api_portal\W+|spring_cloud_api_portal_custom_domain\W+|spring_cloud_app\W+|spring_cloud_app_cosmosdb_association\W+|spring_cloud_app_dynamics_application_performance_monitoring\W+|spring_cloud_app_mysql_association\W+|spring_cloud_app_redis_association\W+|spring_cloud_application_insights_application_performance_monitoring\W+|spring_cloud_application_live_view\W+|spring_cloud_build_deployment\W+|spring_cloud_build_pack_binding\W+|spring_cloud_builder\W+|spring_cloud_certificate\W+|spring_cloud_configuration_service\W+|spring_cloud_container_deployment\W+|spring_cloud_custom_domain\W+|spring_cloud_customized_accelerator\W+|spring_cloud_dev_tool_portal\W+|spring_cloud_dynatrace_application_performance_monitoring\W+|spring_cloud_elastic_application_performance_monitoring\W+|spring_cloud_gateway\W+|spring_cloud_gateway_custom_domain\W+|spring_cloud_gateway_route_config\W+|spring_cloud_java_deployment\W+|spring_cloud_new_relic_application_performance_monitoring\W+|spring_cloud_service\W+|spring_cloud_storage\W+)((.|\n)*)###'

service/sql:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_sql_((.|\n)*)###'

service/storage:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(storage_account\W+|storage_account_blob_container_sas\W+|storage_account_customer_managed_key\W+|storage_account_local_user\W+|storage_account_network_rules\W+|storage_account_sas\W+|storage_blob\W+|storage_blob_inventory_policy\W+|storage_container\W+|storage_container_immutability_policy\W+|storage_containers\W+|storage_data_lake_gen2_filesystem\W+|storage_data_lake_gen2_path\W+|storage_encryption_scope\W+|storage_management_policy\W+|storage_object_replication\W+|storage_queue\W+|storage_share\W+|storage_share_directory\W+|storage_share_file\W+|storage_sync\W+|storage_sync_cloud_endpoint\W+|storage_sync_group\W+|storage_sync_server_endpoint\W+|storage_table\W+|storage_table\W+|storage_table_entities\W+|storage_table_entity\W+)((.|\n)*)###'
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(storage_account\W+|storage_account_blob_container_sas\W+|storage_account_customer_managed_key\W+|storage_account_local_user\W+|storage_account_network_rules\W+|storage_account_queue_properties\W+|storage_account_sas\W+|storage_account_static_website\W+|storage_blob\W+|storage_blob_inventory_policy\W+|storage_container\W+|storage_container_immutability_policy\W+|storage_containers\W+|storage_data_lake_gen2_filesystem\W+|storage_data_lake_gen2_path\W+|storage_encryption_scope\W+|storage_management_policy\W+|storage_object_replication\W+|storage_queue\W+|storage_share\W+|storage_share_directory\W+|storage_share_file\W+|storage_sync\W+|storage_sync_cloud_endpoint\W+|storage_sync_group\W+|storage_sync_server_endpoint\W+|storage_table\W+|storage_table\W+|storage_table_entities\W+|storage_table_entity\W+)((.|\n)*)###'

service/storagemover:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_storage_mover((.|\n)*)###'
Expand All @@ -352,7 +347,6 @@ service/traffic-manager:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_traffic_manager_((.|\n)*)###'

service/video-analyzer:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_video_analyzer((.|\n)*)###'

service/virtual-desktops:
- '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_virtual_desktop_((.|\n)*)###'
Expand Down
14 changes: 14 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
## 3.117.0 (November 7, 2024)

SPECIAL NOTES: This 3.x release is a special, one-off, back-port of functionality for `azurerm_storage_account` to enable users to deploy this resource in environments which block / are restrictive of Data Plane access, thus preventing the resource being created and/or managed. This functionality is back-ported from the `v4.9.0` release. Users migrating from this release to the 4.x line, should upgrade directly to `v4.9.0` or later, as these features are not compatible with earlier releases of 4.x.

FEATURES:

* **New Resource:** `azurerm_storage_account_queue_properties` ([#27819](https://github.com/hashicorp/terraform-provider-azurerm/pull/27819))
* **New Resource:** `azurerm_storage_account_static_website` ([#27819](https://github.com/hashicorp/terraform-provider-azurerm/pull/27819))
* New Provider Feature - storage `data_plane_available` feature flag ([#27819](https://github.com/hashicorp/terraform-provider-azurerm/pull/27819))

ENHANCEMENTS:

* `azurerm_storage_account` - can now be created and managed if Data Plane endpoints are blocked by a firewall ([#27819](https://github.com/hashicorp/terraform-provider-azurerm/pull/27819))

## 3.116.0 (August 16, 2024)

DEPRECATIONS:
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ require (
github.com/google/uuid v1.6.0
github.com/hashicorp/go-azure-helpers v0.70.1
github.com/hashicorp/go-azure-sdk/resource-manager v0.20240731.1212841
github.com/hashicorp/go-azure-sdk/sdk v0.20240731.1212841
github.com/hashicorp/go-azure-sdk/sdk v0.20241025.1143247 // out of sync with rm due to drift of versions but required network fixes
github.com/hashicorp/go-hclog v1.6.3
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/go-uuid v1.0.3
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ github.com/hashicorp/go-azure-helpers v0.70.1 h1:7hlnRrZobMZxpOzdlNEsayzAayj/KRG
github.com/hashicorp/go-azure-helpers v0.70.1/go.mod h1:BmbF4JDYXK5sEmFeU5hcn8Br21uElcqLfdQxjatwQKw=
github.com/hashicorp/go-azure-sdk/resource-manager v0.20240731.1212841 h1:H7BkxZl0qitdWq7sEGzNqkn5/11YTamwq2XTI8/7Jq0=
github.com/hashicorp/go-azure-sdk/resource-manager v0.20240731.1212841/go.mod h1:/4Ly9Gppp/Nu9AaWDfod6atYQ4n2OqN0ERpE2xZQz8A=
github.com/hashicorp/go-azure-sdk/sdk v0.20240731.1212841 h1:RzWuy96j/7q3Vi2aZoiIrokm8yotUNX1UGD3pbWi5Ck=
github.com/hashicorp/go-azure-sdk/sdk v0.20240731.1212841/go.mod h1:dMKF6bXrgGmy1d3pLzkmBpG2JIHgSAV2/OMSCEgyMwE=
github.com/hashicorp/go-azure-sdk/sdk v0.20241025.1143247 h1:NoYFgxtEsxHhE6TyJ6DRXqHLcxZ0cmrpGxNPio0lT84=
github.com/hashicorp/go-azure-sdk/sdk v0.20241025.1143247/go.mod h1:dMKF6bXrgGmy1d3pLzkmBpG2JIHgSAV2/OMSCEgyMwE=
github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU=
github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
Expand Down
2 changes: 1 addition & 1 deletion internal/clients/graph/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ type directoryObjectModel struct {
}

func graphClient(authorizer auth.Authorizer, environment environments.Environment) (*msgraph.Client, error) {
client, err := msgraph.NewMsGraphClient(environment.MicrosoftGraph, "Graph", msgraph.VersionOnePointZero)
client, err := msgraph.NewClient(environment.MicrosoftGraph, "Graph", msgraph.VersionOnePointZero)
if err != nil {
return nil, fmt.Errorf("building client: %+v", err)
}
Expand Down
3 changes: 3 additions & 0 deletions internal/features/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ func Default() UserFeatures {
RollInstancesWhenRequired: true,
ScaleToZeroOnDelete: true,
},
Storage: StorageFeatures{
DataPlaneAvailable: true,
},
Subscription: SubscriptionFeatures{
PreventCancellationOnDestroy: false,
},
Expand Down
5 changes: 5 additions & 0 deletions internal/features/user_flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ type UserFeatures struct {
ResourceGroup ResourceGroupFeatures
RecoveryServicesVault RecoveryServicesVault
ManagedDisk ManagedDiskFeatures
Storage StorageFeatures
Subscription SubscriptionFeatures
PostgresqlFlexibleServer PostgresqlFlexibleServerFeatures
MachineLearning MachineLearningFeatures
Expand Down Expand Up @@ -84,6 +85,10 @@ type AppConfigurationFeatures struct {
RecoverSoftDeleted bool
}

type StorageFeatures struct {
DataPlaneAvailable bool
}

type SubscriptionFeatures struct {
PreventCancellationOnDestroy bool
}
Expand Down
25 changes: 25 additions & 0 deletions internal/provider/features.go
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,21 @@ func schemaFeatures(supportLegacyTestSuite bool) *pluginsdk.Schema {
},
},

"storage": {
Type: pluginsdk.TypeList,
Optional: true,
MaxItems: 1,
Elem: &pluginsdk.Resource{
Schema: map[string]*schema.Schema{
"data_plane_available": {
Type: pluginsdk.TypeBool,
Optional: true,
Default: true,
},
},
},
},

"subscription": {
Type: pluginsdk.TypeList,
Optional: true,
Expand Down Expand Up @@ -581,6 +596,16 @@ func expandFeatures(input []interface{}) features.UserFeatures {
}
}

if raw, ok := val["storage"]; ok {
items := raw.([]interface{})
if len(items) > 0 {
storageRaw := items[0].(map[string]interface{})
if v, ok := storageRaw["data_plane_available"]; ok {
featuresMap.Storage.DataPlaneAvailable = v.(bool)
}
}
}

if raw, ok := val["subscription"]; ok {
items := raw.([]interface{})
if len(items) > 0 {
Expand Down
67 changes: 67 additions & 0 deletions internal/provider/features_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ func TestExpandFeatures(t *testing.T) {
RecoveryServicesVault: features.RecoveryServicesVault{
RecoverSoftDeletedBackupProtectedVM: true,
},
Storage: features.StorageFeatures{
DataPlaneAvailable: true,
},
Subscription: features.SubscriptionFeatures{
PreventCancellationOnDestroy: false,
},
Expand Down Expand Up @@ -156,6 +159,11 @@ func TestExpandFeatures(t *testing.T) {
"recover_soft_deleted_backup_protected_vm": true,
},
},
"storage": []interface{}{
map[string]interface{}{
"data_plane_available": true,
},
},
"subscription": []interface{}{
map[string]interface{}{
"prevent_cancellation_on_destroy": true,
Expand Down Expand Up @@ -235,6 +243,9 @@ func TestExpandFeatures(t *testing.T) {
RecoveryServicesVault: features.RecoveryServicesVault{
RecoverSoftDeletedBackupProtectedVM: true,
},
Storage: features.StorageFeatures{
DataPlaneAvailable: true,
},
Subscription: features.SubscriptionFeatures{
PreventCancellationOnDestroy: true,
},
Expand Down Expand Up @@ -331,6 +342,11 @@ func TestExpandFeatures(t *testing.T) {
"recover_soft_deleted_backup_protected_vm": false,
},
},
"storage": []interface{}{
map[string]interface{}{
"data_plane_available": false,
},
},
"subscription": []interface{}{
map[string]interface{}{
"prevent_cancellation_on_destroy": false,
Expand Down Expand Up @@ -410,6 +426,9 @@ func TestExpandFeatures(t *testing.T) {
RecoveryServicesVault: features.RecoveryServicesVault{
RecoverSoftDeletedBackupProtectedVM: false,
},
Storage: features.StorageFeatures{
DataPlaneAvailable: false,
},
Subscription: features.SubscriptionFeatures{
PreventCancellationOnDestroy: false,
},
Expand Down Expand Up @@ -1431,6 +1450,54 @@ func TestExpandFeaturesManagedDisk(t *testing.T) {
}
}

func TestExpandFeaturesStorage(t *testing.T) {
testData := []struct {
Name string
Input []interface{}
EnvVars map[string]interface{}
Expected features.UserFeatures
}{
{
Name: "Empty Block",
Input: []interface{}{
map[string]interface{}{
"storage": []interface{}{},
},
},
Expected: features.UserFeatures{
Storage: features.StorageFeatures{
DataPlaneAvailable: true,
},
},
},
{
Name: "Storage Data Plane on Create is Disabled",
Input: []interface{}{
map[string]interface{}{
"storage": []interface{}{
map[string]interface{}{
"data_plane_available": false,
},
},
},
},
Expected: features.UserFeatures{
Storage: features.StorageFeatures{
DataPlaneAvailable: false,
},
},
},
}

for _, testCase := range testData {
t.Logf("[DEBUG] Test Case: %q", testCase.Name)
result := expandFeatures(testCase.Input)
if !reflect.DeepEqual(result.Storage, testCase.Expected.Storage) {
t.Fatalf("Expected %+v but got %+v", result.Storage, testCase.Expected.Storage)
}
}
}

func TestExpandFeaturesSubscription(t *testing.T) {
testData := []struct {
Name string
Expand Down
13 changes: 13 additions & 0 deletions internal/provider/framework/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,19 @@ func (p *ProviderConfig) Load(ctx context.Context, data *ProviderModel, tfVersio
f.ManagedDisk.ExpandWithoutDowntime = true
}

if !features.Storage.IsNull() && !features.Storage.IsUnknown() {
var feature []Storage
d := features.Storage.ElementsAs(ctx, &feature, true)
diags.Append(d...)
if diags.HasError() {
return
}
f.Storage.DataPlaneAvailable = true
if !feature[0].DataPlaneAvailable.IsNull() && !feature[0].DataPlaneAvailable.IsUnknown() {
f.Storage.DataPlaneAvailable = feature[0].DataPlaneAvailable.ValueBool()
}
}

if !features.Subscription.IsNull() && !features.Subscription.IsUnknown() {
var feature []Subscription
d := features.Subscription.ElementsAs(ctx, &feature, true)
Expand Down
Loading

0 comments on commit 7e34ba7

Please sign in to comment.