Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Storage integration with custom protocol #3213

Merged
merged 4 commits into from
Nov 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/resources/storage_integration.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ resource "snowflake_storage_integration" "integration" {

- `name` (String)
- `storage_allowed_locations` (List of String) Explicitly limits external stages that use the integration to reference one or more storage locations.
- `storage_provider` (String)
- `storage_provider` (String) Specifies the storage provider for the integration. Valid options are: `S3` | `S3GOV` | `S3CHINA` | `GCS` | `AZURE`

### Optional

Expand Down
8 changes: 4 additions & 4 deletions pkg/acceptance/helpers/storage_integration_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ func (c *StorageIntegrationClient) CreateS3(t *testing.T, awsBucketUrl, awsRoleA

id := c.ids.RandomAccountObjectIdentifier()
req := sdk.NewCreateStorageIntegrationRequest(id, true, s3AllowedLocations).
WithIfNotExists(sdk.Bool(true)).
WithS3StorageProviderParams(sdk.NewS3StorageParamsRequest(awsRoleArn)).
WithIfNotExists(true).
WithS3StorageProviderParams(*sdk.NewS3StorageParamsRequest(sdk.RegularS3Protocol, awsRoleArn)).
WithStorageBlockedLocations(s3BlockedLocations).
WithComment(sdk.String("some comment"))
WithComment("some comment")

err := c.client().Create(ctx, req)
require.NoError(t, err)
Expand All @@ -73,7 +73,7 @@ func (c *StorageIntegrationClient) DropFunc(t *testing.T, id sdk.AccountObjectId
ctx := context.Background()

return func() {
err := c.client().Drop(ctx, sdk.NewDropStorageIntegrationRequest(id).WithIfExists(sdk.Bool(true)))
err := c.client().Drop(ctx, sdk.NewDropStorageIntegrationRequest(id).WithIfExists(true))
require.NoError(t, err)
}
}
52 changes: 29 additions & 23 deletions pkg/resources/storage_integration.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log"
"slices"
"strings"

"github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/internal/provider"
Expand Down Expand Up @@ -52,12 +53,12 @@ var storageIntegrationSchema = map[string]*schema.Schema{
Optional: true,
Description: "Explicitly prohibits external stages that use the integration from referencing one or more storage locations.",
},
// TODO (SNOW-1015282): Remove S3gov option before going into V1
"storage_provider": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
ValidateFunc: validation.StringInSlice([]string{"S3", "S3gov", "GCS", "AZURE", "S3GOV"}, false),
Type: schema.TypeString,
Required: true,
ForceNew: true,
ValidateDiagFunc: StringInSlice(sdk.AllStorageProviders, true),
Description: fmt.Sprintf("Specifies the storage provider for the integration. Valid options are: %s", possibleValuesListed(sdk.AllStorageProviders)),
},
"storage_aws_external_id": {
Type: schema.TypeString,
Expand Down Expand Up @@ -140,7 +141,7 @@ func CreateStorageIntegration(d *schema.ResourceData, meta any) error {
req := sdk.NewCreateStorageIntegrationRequest(name, enabled, storageAllowedLocations)

if v, ok := d.GetOk("comment"); ok {
req.WithComment(sdk.String(v.(string)))
req.WithComment(v.(string))
}

if _, ok := d.GetOk("storage_blocked_locations"); ok {
Expand All @@ -154,28 +155,33 @@ func CreateStorageIntegration(d *schema.ResourceData, meta any) error {
req.WithStorageBlockedLocations(storageBlockedLocations)
}

storageProvider := d.Get("storage_provider").(string)
storageProvider := strings.ToUpper(d.Get("storage_provider").(string))
sfc-gh-jmichalak marked this conversation as resolved.
Show resolved Hide resolved

switch {
case slices.Contains(sdk.AllS3Protocols, sdk.S3Protocol(storageProvider)):
s3Protocol, err := sdk.ToS3Protocol(storageProvider)
if err != nil {
return err
}

switch storageProvider {
case "S3", "S3GOV", "S3gov":
v, ok := d.GetOk("storage_aws_role_arn")
if !ok {
return fmt.Errorf("if you use the S3 storage provider you must specify a storage_aws_role_arn")
}

s3Params := sdk.NewS3StorageParamsRequest(v.(string))
s3Params := sdk.NewS3StorageParamsRequest(s3Protocol, v.(string))
if _, ok := d.GetOk("storage_aws_object_acl"); ok {
s3Params.WithStorageAwsObjectAcl(sdk.String(d.Get("storage_aws_object_acl").(string)))
s3Params.WithStorageAwsObjectAcl(d.Get("storage_aws_object_acl").(string))
}
req.WithS3StorageProviderParams(s3Params)
case "AZURE":
req.WithS3StorageProviderParams(*s3Params)
case storageProvider == "AZURE":
v, ok := d.GetOk("azure_tenant_id")
if !ok {
return fmt.Errorf("if you use the Azure storage provider you must specify an azure_tenant_id")
}
req.WithAzureStorageProviderParams(sdk.NewAzureStorageParamsRequest(sdk.String(v.(string))))
case "GCS":
req.WithGCSStorageProviderParams(sdk.NewGCSStorageParamsRequest())
req.WithAzureStorageProviderParams(*sdk.NewAzureStorageParamsRequest(sdk.String(v.(string))))
case storageProvider == "GCS":
req.WithGCSStorageProviderParams(*sdk.NewGCSStorageParamsRequest())
default:
return fmt.Errorf("unexpected provider %v", storageProvider)
}
Expand Down Expand Up @@ -295,7 +301,7 @@ func UpdateStorageIntegration(d *schema.ResourceData, meta any) error {

if d.HasChange("comment") {
runSetStatement = true
setReq.WithComment(sdk.String(d.Get("comment").(string)))
setReq.WithComment(d.Get("comment").(string))
}

if d.HasChange("enabled") {
Expand All @@ -320,7 +326,7 @@ func UpdateStorageIntegration(d *schema.ResourceData, meta any) error {
v := d.Get("storage_blocked_locations").([]interface{})
if len(v) == 0 {
if err := client.StorageIntegrations.Alter(ctx, sdk.NewAlterStorageIntegrationRequest(id).
WithUnset(sdk.NewStorageIntegrationUnsetRequest().WithStorageBlockedLocations(sdk.Bool(true)))); err != nil {
WithUnset(*sdk.NewStorageIntegrationUnsetRequest().WithStorageBlockedLocations(true))); err != nil {
return fmt.Errorf("error unsetting storage_blocked_locations, err = %w", err)
}
} else {
Expand All @@ -342,25 +348,25 @@ func UpdateStorageIntegration(d *schema.ResourceData, meta any) error {

if d.HasChange("storage_aws_object_acl") {
if v, ok := d.GetOk("storage_aws_object_acl"); ok {
s3SetParams.WithStorageAwsObjectAcl(sdk.String(v.(string)))
s3SetParams.WithStorageAwsObjectAcl(v.(string))
} else {
if err := client.StorageIntegrations.Alter(ctx, sdk.NewAlterStorageIntegrationRequest(id).
WithUnset(sdk.NewStorageIntegrationUnsetRequest().WithStorageAwsObjectAcl(sdk.Bool(true)))); err != nil {
WithUnset(*sdk.NewStorageIntegrationUnsetRequest().WithStorageAwsObjectAcl(true))); err != nil {
return fmt.Errorf("error unsetting storage_aws_object_acl, err = %w", err)
}
}
}

setReq.WithS3Params(s3SetParams)
setReq.WithS3Params(*s3SetParams)
}

if d.HasChange("azure_tenant_id") {
runSetStatement = true
setReq.WithAzureParams(sdk.NewSetAzureStorageParamsRequest(d.Get("azure_tenant_id").(string)))
setReq.WithAzureParams(*sdk.NewSetAzureStorageParamsRequest(d.Get("azure_tenant_id").(string)))
}

if runSetStatement {
if err := client.StorageIntegrations.Alter(ctx, sdk.NewAlterStorageIntegrationRequest(id).WithSet(setReq)); err != nil {
if err := client.StorageIntegrations.Alter(ctx, sdk.NewAlterStorageIntegrationRequest(id).WithSet(*setReq)); err != nil {
return fmt.Errorf("error updating storage integration, err = %w", err)
}
}
Expand Down
33 changes: 30 additions & 3 deletions pkg/sdk/storage_integration_def.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,36 @@
package sdk

import g "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk/poc/generator"
import (
"fmt"
"strings"

g "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk/poc/generator"
)

//go:generate go run ./poc/main.go

type S3Protocol string

const (
RegularS3Protocol S3Protocol = "S3"
GovS3Protocol S3Protocol = "S3GOV"
ChinaS3Protocol S3Protocol = "S3CHINA"
)

var (
AllS3Protocols = []S3Protocol{RegularS3Protocol, GovS3Protocol, ChinaS3Protocol}
AllStorageProviders = append(AsStringList(AllS3Protocols), "GCS", "AZURE")
)

func ToS3Protocol(s string) (S3Protocol, error) {
switch protocol := S3Protocol(strings.ToUpper(s)); protocol {
case RegularS3Protocol, GovS3Protocol, ChinaS3Protocol:
return protocol, nil
default:
return "", fmt.Errorf("invalid S3 protocol: %s", s)
}
}

var StorageLocationDef = g.NewQueryStruct("StorageLocation").Text("Path", g.KeywordOptions().SingleQuotes().Required())

var StorageIntegrationDef = g.NewInterface(
Expand All @@ -23,7 +50,7 @@ var StorageIntegrationDef = g.NewInterface(
OptionalQueryStructField(
"S3StorageProviderParams",
g.NewQueryStruct("S3StorageParams").
PredefinedQueryStructField("storageProvider", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'S3'")).
PredefinedQueryStructField("Protocol", g.KindOfT[S3Protocol](), g.ParameterOptions().SQL("STORAGE_PROVIDER").SingleQuotes().Required()).
TextAssignment("STORAGE_AWS_ROLE_ARN", g.ParameterOptions().SingleQuotes().Required()).
OptionalTextAssignment("STORAGE_AWS_OBJECT_ACL", g.ParameterOptions().SingleQuotes()),
g.KeywordOptions(),
Expand Down Expand Up @@ -73,7 +100,7 @@ var StorageIntegrationDef = g.NewInterface(
TextAssignment("AZURE_TENANT_ID", g.ParameterOptions().SingleQuotes().Required()),
g.KeywordOptions(),
).
BooleanAssignment("ENABLED", g.ParameterOptions()).
OptionalBooleanAssignment("ENABLED", g.ParameterOptions()).
ListAssignment("STORAGE_ALLOWED_LOCATIONS", "StorageLocation", g.ParameterOptions().Parentheses()).
ListAssignment("STORAGE_BLOCKED_LOCATIONS", "StorageLocation", g.ParameterOptions().Parentheses()).
OptionalComment(),
Expand Down
Loading
Loading