Skip to content

Commit

Permalink
Merge pull request #16052 from DrFaust92/r/glue_table_validations
Browse files Browse the repository at this point in the history 
r/glue_catalog_table - add `parameters` argument to `columns` + validations
  • Loading branch information
@breathingdust
breathingdust authored Nov 17, 2020
2 parents 0a6fe66 + be4ef49 commit 02092c2
Show file tree
Hide file tree
Showing 3 changed files with 223 additions and 97 deletions.
184 changes: 96 additions & 88 deletions aws/resource_aws_glue_catalog_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package aws
import (
"fmt"
"log"
"regexp"
"strings"

"github.com/aws/aws-sdk-go/aws"
Expand Down Expand Up @@ -39,13 +40,18 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
Required: true,
},
"description": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 2048),
},
"name": {
Type: schema.TypeString,
ForceNew: true,
Required: true,
ValidateFunc: validation.All(
validation.StringLenBetween(1, 255),
validation.StringDoesNotMatch(regexp.MustCompile(`[A-Z]`), "uppercase characters cannot be used"),
),
},
"owner": {
Type: schema.TypeString,
Expand All @@ -62,23 +68,27 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"comment": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 255),
},
"name": {
Type: schema.TypeString,
Required: true,
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(1, 255),
},
"type": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 131072),
},
},
},
},
"retention": {
Type: schema.TypeInt,
Optional: true,
Type: schema.TypeInt,
Optional: true,
ValidateFunc: validation.IntAtLeast(0),
},
"storage_descriptor": {
Type: schema.TypeList,
Expand All @@ -89,24 +99,35 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
"bucket_columns": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Schema{Type: schema.TypeString},
Elem: &schema.Schema{
Type: schema.TypeString,
ValidateFunc: validation.StringLenBetween(1, 255),
},
},
"columns": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"comment": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 255),
},
"name": {
Type: schema.TypeString,
Required: true,
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(1, 255),
},
"type": {
Type: schema.TypeString,
"parameters": {
Type: schema.TypeMap,
Optional: true,
Elem: &schema.Schema{Type: schema.TypeString},
},
"type": {
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 131072),
},
},
},
Expand Down Expand Up @@ -145,7 +166,7 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
"name": {
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringIsNotEmpty,
ValidateFunc: validation.StringLenBetween(1, 255),
},
"parameters": {
Type: schema.TypeMap,
Expand All @@ -169,7 +190,10 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
"skewed_column_names": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Schema{Type: schema.TypeString},
Elem: &schema.Schema{
Type: schema.TypeString,
ValidateFunc: validation.StringLenBetween(1, 255),
},
},
"skewed_column_values": {
Type: schema.TypeList,
Expand All @@ -190,12 +214,14 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"column": {
Type: schema.TypeString,
Required: true,
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(1, 255),
},
"sort_order": {
Type: schema.TypeInt,
Required: true,
Type: schema.TypeInt,
Required: true,
ValidateFunc: validation.IntInSlice([]int{0, 1}),
},
},
},
Expand All @@ -212,12 +238,14 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
Optional: true,
},
"view_original_text": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 409600),
},
"view_expanded_text": {
Type: schema.TypeString,
Optional: true,
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 409600),
},
},
}
Expand All @@ -243,9 +271,10 @@ func resourceAwsGlueCatalogTableCreate(d *schema.ResourceData, meta interface{})
TableInput: expandGlueTableInput(d),
}

log.Printf("[DEBUG] Glue catalog table input: %#v", input)
_, err := conn.CreateTable(input)
if err != nil {
return fmt.Errorf("Error creating Catalog Table: %s", err)
return fmt.Errorf("Error creating Glue Catalog Table: %w", err)
}

d.SetId(fmt.Sprintf("%s:%s:%s", catalogID, dbName, name))
Expand Down Expand Up @@ -276,39 +305,40 @@ func resourceAwsGlueCatalogTableRead(d *schema.ResourceData, meta interface{}) e
return nil
}

return fmt.Errorf("Error reading Glue Catalog Table: %s", err)
return fmt.Errorf("Error reading Glue Catalog Table: %w", err)
}

table := out.Table
tableArn := arn.ARN{
Partition: meta.(*AWSClient).partition,
Service: "glue",
Region: meta.(*AWSClient).region,
AccountID: meta.(*AWSClient).accountid,
Resource: fmt.Sprintf("table/%s/%s", dbName, aws.StringValue(out.Table.Name)),
Resource: fmt.Sprintf("table/%s/%s", dbName, aws.StringValue(table.Name)),
}.String()
d.Set("arn", tableArn)

d.Set("name", out.Table.Name)
d.Set("name", table.Name)
d.Set("catalog_id", catalogID)
d.Set("database_name", dbName)
d.Set("description", out.Table.Description)
d.Set("owner", out.Table.Owner)
d.Set("retention", out.Table.Retention)
d.Set("description", table.Description)
d.Set("owner", table.Owner)
d.Set("retention", table.Retention)

if err := d.Set("storage_descriptor", flattenGlueStorageDescriptor(out.Table.StorageDescriptor)); err != nil {
return fmt.Errorf("error setting storage_descriptor: %s", err)
if err := d.Set("storage_descriptor", flattenGlueStorageDescriptor(table.StorageDescriptor)); err != nil {
return fmt.Errorf("error setting storage_descriptor: %w", err)
}

if err := d.Set("partition_keys", flattenGlueColumns(out.Table.PartitionKeys)); err != nil {
return fmt.Errorf("error setting partition_keys: %s", err)
if err := d.Set("partition_keys", flattenGlueColumns(table.PartitionKeys)); err != nil {
return fmt.Errorf("error setting partition_keys: %w", err)
}

d.Set("view_original_text", out.Table.ViewOriginalText)
d.Set("view_expanded_text", out.Table.ViewExpandedText)
d.Set("table_type", out.Table.TableType)
d.Set("view_original_text", table.ViewOriginalText)
d.Set("view_expanded_text", table.ViewExpandedText)
d.Set("table_type", table.TableType)

if err := d.Set("parameters", aws.StringValueMap(out.Table.Parameters)); err != nil {
return fmt.Errorf("error setting parameters: %s", err)
if err := d.Set("parameters", aws.StringValueMap(table.Parameters)); err != nil {
return fmt.Errorf("error setting parameters: %w", err)
}

return nil
Expand All @@ -329,7 +359,7 @@ func resourceAwsGlueCatalogTableUpdate(d *schema.ResourceData, meta interface{})
}

if _, err := conn.UpdateTable(updateTableInput); err != nil {
return fmt.Errorf("Error updating Glue Catalog Table: %s", err)
return fmt.Errorf("Error updating Glue Catalog Table: %w", err)
}

return resourceAwsGlueCatalogTableRead(d, meta)
Expand All @@ -350,7 +380,7 @@ func resourceAwsGlueCatalogTableDelete(d *schema.ResourceData, meta interface{})
DatabaseName: aws.String(dbName),
})
if err != nil {
return fmt.Errorf("Error deleting Glue Catalog Table: %s", err.Error())
return fmt.Errorf("Error deleting Glue Catalog Table: %w", err)
}
return nil
}
Expand All @@ -377,8 +407,7 @@ func expandGlueTableInput(d *schema.ResourceData) *glue.TableInput {
}

if v, ok := d.GetOk("partition_keys"); ok {
columns := expandGlueColumns(v.([]interface{}))
tableInput.PartitionKeys = columns
tableInput.PartitionKeys = expandGlueColumns(v.([]interface{}))
}

if v, ok := d.GetOk("view_original_text"); ok {
Expand All @@ -394,11 +423,7 @@ func expandGlueTableInput(d *schema.ResourceData) *glue.TableInput {
}

if v, ok := d.GetOk("parameters"); ok {
paramsMap := map[string]string{}
for key, value := range v.(map[string]interface{}) {
paramsMap[key] = value.(string)
}
tableInput.Parameters = aws.StringMap(paramsMap)
tableInput.Parameters = stringMapToPointers(v.(map[string]interface{}))
}

return tableInput
Expand All @@ -413,8 +438,7 @@ func expandGlueStorageDescriptor(l []interface{}) *glue.StorageDescriptor {
storageDescriptor := &glue.StorageDescriptor{}

if v, ok := s["columns"]; ok {
columns := expandGlueColumns(v.([]interface{}))
storageDescriptor.Columns = columns
storageDescriptor.Columns = expandGlueColumns(v.([]interface{}))
}

if v, ok := s["location"]; ok {
Expand Down Expand Up @@ -442,11 +466,7 @@ func expandGlueStorageDescriptor(l []interface{}) *glue.StorageDescriptor {
}

if v, ok := s["bucket_columns"]; ok {
bucketColumns := make([]string, len(v.([]interface{})))
for i, item := range v.([]interface{}) {
bucketColumns[i] = fmt.Sprint(item)
}
storageDescriptor.BucketColumns = aws.StringSlice(bucketColumns)
storageDescriptor.BucketColumns = expandStringList(v.([]interface{}))
}

if v, ok := s["sort_columns"]; ok {
Expand All @@ -458,11 +478,7 @@ func expandGlueStorageDescriptor(l []interface{}) *glue.StorageDescriptor {
}

if v, ok := s["parameters"]; ok {
paramsMap := map[string]string{}
for key, value := range v.(map[string]interface{}) {
paramsMap[key] = value.(string)
}
storageDescriptor.Parameters = aws.StringMap(paramsMap)
storageDescriptor.Parameters = stringMapToPointers(v.(map[string]interface{}))
}

if v, ok := s["stored_as_sub_directories"]; ok {
Expand All @@ -489,6 +505,10 @@ func expandGlueColumns(columns []interface{}) []*glue.Column {
column.Type = aws.String(v.(string))
}

if v, ok := elementMap["parameters"]; ok {
column.Parameters = stringMapToPointers(v.(map[string]interface{}))
}

columnSlice = append(columnSlice, column)
}

Expand All @@ -508,11 +528,7 @@ func expandGlueSerDeInfo(l []interface{}) *glue.SerDeInfo {
}

if v := s["parameters"]; len(v.(map[string]interface{})) > 0 {
paramsMap := map[string]string{}
for key, value := range v.(map[string]interface{}) {
paramsMap[key] = value.(string)
}
serDeInfo.Parameters = aws.StringMap(paramsMap)
serDeInfo.Parameters = stringMapToPointers(v.(map[string]interface{}))
}

if v := s["serialization_library"]; len(v.(string)) > 0 {
Expand Down Expand Up @@ -551,27 +567,15 @@ func expandGlueSkewedInfo(l []interface{}) *glue.SkewedInfo {
skewedInfo := &glue.SkewedInfo{}

if v, ok := s["skewed_column_names"]; ok {
columnsSlice := make([]string, len(v.([]interface{})))
for i, item := range v.([]interface{}) {
columnsSlice[i] = fmt.Sprint(item)
}
skewedInfo.SkewedColumnNames = aws.StringSlice(columnsSlice)
skewedInfo.SkewedColumnNames = expandStringList(v.([]interface{}))
}

if v, ok := s["skewed_column_value_location_maps"]; ok {
typeMap := map[string]string{}
for key, value := range v.(map[string]interface{}) {
typeMap[key] = value.(string)
}
skewedInfo.SkewedColumnValueLocationMaps = aws.StringMap(typeMap)
skewedInfo.SkewedColumnValueLocationMaps = stringMapToPointers(v.(map[string]interface{}))
}

if v, ok := s["skewed_column_values"]; ok {
columnsSlice := make([]string, len(v.([]interface{})))
for i, item := range v.([]interface{}) {
columnsSlice[i] = fmt.Sprint(item)
}
skewedInfo.SkewedColumnValues = aws.StringSlice(columnsSlice)
skewedInfo.SkewedColumnValues = expandStringList(v.([]interface{}))
}

return skewedInfo
Expand Down Expand Up @@ -605,8 +609,8 @@ func flattenGlueStorageDescriptor(s *glue.StorageDescriptor) []map[string]interf
return storageDescriptors
}

func flattenGlueColumns(cs []*glue.Column) []map[string]string {
columnsSlice := make([]map[string]string, len(cs))
func flattenGlueColumns(cs []*glue.Column) []map[string]interface{} {
columnsSlice := make([]map[string]interface{}, len(cs))
if len(cs) > 0 {
for i, v := range cs {
columnsSlice[i] = flattenGlueColumn(v)
Expand All @@ -616,8 +620,8 @@ func flattenGlueColumns(cs []*glue.Column) []map[string]string {
return columnsSlice
}

func flattenGlueColumn(c *glue.Column) map[string]string {
column := make(map[string]string)
func flattenGlueColumn(c *glue.Column) map[string]interface{} {
column := make(map[string]interface{})

if c == nil {
return column
Expand All @@ -635,6 +639,10 @@ func flattenGlueColumn(c *glue.Column) map[string]string {
column["comment"] = v
}

if v := c.Parameters; v != nil {
column["parameters"] = aws.StringValueMap(v)
}

return column
}

Expand Down
Loading

0 comments on commit 02092c2

Please sign in to comment.