diff --git a/aws/resource_aws_glue_catalog_table.go b/aws/resource_aws_glue_catalog_table.go index e803e449255..8cb3669351e 100644 --- a/aws/resource_aws_glue_catalog_table.go +++ b/aws/resource_aws_glue_catalog_table.go @@ -219,6 +219,31 @@ func resourceAwsGlueCatalogTable() *schema.Resource { Type: schema.TypeString, Optional: true, }, + "partition_index": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + MaxItems: 3, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "index_name": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringLenBetween(1, 255), + }, + "keys": { + Type: schema.TypeSet, + Required: true, + MinItems: 1, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "index_status": { + Type: schema.TypeString, + Computed: true, + }, + }, + }, + }, }, } } @@ -238,9 +263,10 @@ func resourceAwsGlueCatalogTableCreate(d *schema.ResourceData, meta interface{}) name := d.Get("name").(string) input := &glue.CreateTableInput{ - CatalogId: aws.String(catalogID), - DatabaseName: aws.String(dbName), - TableInput: expandGlueTableInput(d), + CatalogId: aws.String(catalogID), + DatabaseName: aws.String(dbName), + TableInput: expandGlueTableInput(d), + PartitionIndexes: expandGlueTablePartitionIndexes(d.Get("partition_index").([]interface{})), } _, err := conn.CreateTable(input) @@ -311,6 +337,22 @@ func resourceAwsGlueCatalogTableRead(d *schema.ResourceData, meta interface{}) e return fmt.Errorf("error setting parameters: %s", err) } + partIndexInput := &glue.GetPartitionIndexesInput{ + CatalogId: out.Table.CatalogId, + TableName: out.Table.Name, + DatabaseName: out.Table.DatabaseName, + } + partOut, err := conn.GetPartitionIndexes(partIndexInput) + if err != nil { + return fmt.Errorf("error getting Glue Partition Indexes: %w", err) + } + + if partOut != nil && len(partOut.PartitionIndexDescriptorList) > 0 { + if err := d.Set("partition_index", flattenGluePartitionIndexes(partOut.PartitionIndexDescriptorList)); err != nil { + return fmt.Errorf("error setting partition_index: %w", err) + } + } + return nil } @@ -404,6 +446,25 @@ func expandGlueTableInput(d *schema.ResourceData) *glue.TableInput { return tableInput } +func expandGlueTablePartitionIndexes(a []interface{}) []*glue.PartitionIndex { + partitionIndexes := make([]*glue.PartitionIndex, 0, len(a)) + + for _, m := range a { + partitionIndexes = append(partitionIndexes, expandGlueTablePartitionIndex(m.(map[string]interface{}))) + } + + return partitionIndexes +} + +func expandGlueTablePartitionIndex(m map[string]interface{}) *glue.PartitionIndex { + partitionIndex := &glue.PartitionIndex{ + IndexName: aws.String(m["index_name"].(string)), + Keys: expandStringSet(m["keys"].(*schema.Set)), + } + + return partitionIndex +} + func expandGlueStorageDescriptor(l []interface{}) *glue.StorageDescriptor { if len(l) == 0 || l[0] == nil { return nil @@ -638,6 +699,43 @@ func flattenGlueColumn(c *glue.Column) map[string]string { return column } +func flattenGluePartitionIndexes(cs []*glue.PartitionIndexDescriptor) []map[string]interface{} { + partitionIndexSlice := make([]map[string]interface{}, len(cs)) + if len(cs) > 0 { + for i, v := range cs { + partitionIndexSlice[i] = flattenGluePartitionIndex(v) + } + } + + return partitionIndexSlice +} + +func flattenGluePartitionIndex(c *glue.PartitionIndexDescriptor) map[string]interface{} { + partitionIndex := make(map[string]interface{}) + + if c == nil { + return partitionIndex + } + + if v := aws.StringValue(c.IndexName); v != "" { + partitionIndex["index_name"] = v + } + + if v := aws.StringValue(c.IndexStatus); v != "" { + partitionIndex["index_status"] = v + } + + if c.Keys != nil { + names := make([]*string, 0, len(c.Keys)) + for _, key := range c.Keys { + names = append(names, key.Name) + } + partitionIndex["keys"] = flattenStringSet(names) + } + + return partitionIndex +} + func flattenGlueSerDeInfo(s *glue.SerDeInfo) []map[string]interface{} { if s == nil { serDeInfos := make([]map[string]interface{}, 0) diff --git a/aws/resource_aws_glue_catalog_table_test.go b/aws/resource_aws_glue_catalog_table_test.go index 12914ad5baf..f5b6c1742ba 100644 --- a/aws/resource_aws_glue_catalog_table_test.go +++ b/aws/resource_aws_glue_catalog_table_test.go @@ -427,6 +427,69 @@ func TestAccAWSGlueCatalogTable_StorageDescriptor_SkewedInfo_EmptyConfigurationB }) } +func TestAccAWSGlueCatalogTable_partitionIndexesSingle(t *testing.T) { + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_glue_catalog_table.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckGlueTableDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueCatalogTablePartitionIndexesSingle(rName), + Destroy: false, + Check: resource.ComposeTestCheckFunc( + testAccCheckGlueCatalogTableExists(resourceName), + testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("table/%s/%s", rName, rName)), + resource.TestCheckResourceAttr(resourceName, "partition_index.#", "1"), + resource.TestCheckResourceAttr(resourceName, "partition_index.0.index_name", rName), + resource.TestCheckResourceAttr(resourceName, "partition_index.0.index_status", "ACTIVE"), + resource.TestCheckResourceAttr(resourceName, "partition_index.0.keys.#", "2"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccAWSGlueCatalogTable_partitionIndexesMultiple(t *testing.T) { + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_glue_catalog_table.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckGlueTableDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueCatalogTablePartitionIndexesMultiple(rName), + Destroy: false, + Check: resource.ComposeTestCheckFunc( + testAccCheckGlueCatalogTableExists(resourceName), + testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("table/%s/%s", rName, rName)), + resource.TestCheckResourceAttr(resourceName, "partition_index.#", "2"), + resource.TestCheckResourceAttr(resourceName, "partition_index.0.index_name", rName), + resource.TestCheckResourceAttr(resourceName, "partition_index.0.index_status", "ACTIVE"), + resource.TestCheckResourceAttr(resourceName, "partition_index.0.keys.#", "2"), + resource.TestCheckResourceAttr(resourceName, "partition_index.1.index_name", fmt.Sprintf("%s-2", rName)), + resource.TestCheckResourceAttr(resourceName, "partition_index.1.index_status", "ACTIVE"), + resource.TestCheckResourceAttr(resourceName, "partition_index.1.keys.#", "1"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func TestAccAWSGlueCatalogTable_disappears(t *testing.T) { rName := acctest.RandomWithPrefix("tf-acc-test") resourceName := "aws_glue_catalog_table.test" @@ -812,3 +875,196 @@ func testAccCheckGlueCatalogTableExists(name string) resource.TestCheckFunc { return nil } } + +func testAccGlueCatalogTablePartitionIndexesSingle(rName string) string { + return fmt.Sprintf(` +resource "aws_glue_catalog_database" "test" { + name = %[1]q +} + +resource "aws_glue_catalog_table" "test" { + name = %[1]q + database_name = aws_glue_catalog_database.test.name + owner = "my_owner" + retention = 1 + table_type = "VIRTUAL_VIEW" + view_expanded_text = "view_expanded_text_1" + view_original_text = "view_original_text_1" + + storage_descriptor { + bucket_columns = ["bucket_column_1"] + compressed = false + input_format = "SequenceFileInputFormat" + location = "my_location" + number_of_buckets = 1 + output_format = "SequenceFileInputFormat" + stored_as_sub_directories = false + + parameters = { + param1 = "param1_val" + } + + columns { + name = "my_column_1" + type = "int" + comment = "my_column1_comment" + } + + columns { + name = "my_column_2" + type = "string" + comment = "my_column2_comment" + } + + ser_de_info { + name = "ser_de_name" + + parameters = { + param1 = "param_val_1" + } + + serialization_library = "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" + } + + sort_columns { + column = "my_column_1" + sort_order = 1 + } + + skewed_info { + skewed_column_names = [ + "my_column_1", + ] + + skewed_column_value_location_maps = { + my_column_1 = "my_column_1_val_loc_map" + } + + skewed_column_values = [ + "skewed_val_1", + ] + } + } + + partition_keys { + name = "my_column_1" + type = "int" + comment = "my_column_1_comment" + } + + partition_keys { + name = "my_column_2" + type = "string" + comment = "my_column_2_comment" + } + + parameters = { + param1 = "param1_val" + } + + partition_index { + index_name = %[1]q + keys = ["my_column_1", "my_column_2"] + } +} +`, rName) +} + +func testAccGlueCatalogTablePartitionIndexesMultiple(rName string) string { + return fmt.Sprintf(` +resource "aws_glue_catalog_database" "test" { + name = %[1]q +} + +resource "aws_glue_catalog_table" "test" { + name = %[1]q + database_name = aws_glue_catalog_database.test.name + owner = "my_owner" + retention = 1 + table_type = "VIRTUAL_VIEW" + view_expanded_text = "view_expanded_text_1" + view_original_text = "view_original_text_1" + + storage_descriptor { + bucket_columns = ["bucket_column_1"] + compressed = false + input_format = "SequenceFileInputFormat" + location = "my_location" + number_of_buckets = 1 + output_format = "SequenceFileInputFormat" + stored_as_sub_directories = false + + parameters = { + param1 = "param1_val" + } + + columns { + name = "my_column_1" + type = "int" + comment = "my_column1_comment" + } + + columns { + name = "my_column_2" + type = "string" + comment = "my_column2_comment" + } + + ser_de_info { + name = "ser_de_name" + + parameters = { + param1 = "param_val_1" + } + + serialization_library = "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" + } + + sort_columns { + column = "my_column_1" + sort_order = 1 + } + + skewed_info { + skewed_column_names = [ + "my_column_1", + ] + + skewed_column_value_location_maps = { + my_column_1 = "my_column_1_val_loc_map" + } + + skewed_column_values = [ + "skewed_val_1", + ] + } + } + + partition_keys { + name = "my_column_1" + type = "int" + comment = "my_column_1_comment" + } + + partition_keys { + name = "my_column_2" + type = "string" + comment = "my_column_2_comment" + } + + parameters = { + param1 = "param1_val" + } + + partition_index { + index_name = %[1]q + keys = ["my_column_1", "my_column_2"] + } + + partition_index { + index_name = "%[1]s-2" + keys = ["my_column_1"] + } +} +`, rName) +} diff --git a/website/docs/r/glue_catalog_table.html.markdown b/website/docs/r/glue_catalog_table.html.markdown index af6814b6249..13e6d64e622 100644 --- a/website/docs/r/glue_catalog_table.html.markdown +++ b/website/docs/r/glue_catalog_table.html.markdown @@ -96,6 +96,12 @@ The following arguments are supported: * `view_expanded_text` - (Optional) If the table is a view, the expanded text of the view; otherwise null. * `table_type` - (Optional) The type of this table (EXTERNAL_TABLE, VIRTUAL_VIEW, etc.). While optional, some Athena DDL queries such as `ALTER TABLE` and `SHOW CREATE TABLE` will fail if this argument is empty. * `parameters` - (Optional) Properties associated with this table, as a list of key-value pairs. +* `partition_index` - (Optional) A list of partition indexes. see [Partition Index](#partition-index) below. + +### Partition Index + +* `index_name` - (Required) The name of the partition index. +* `keys` - (Required) The keys for the partition index. ##### storage_descriptor