Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

resource/aws_glue_job: Add tags argument and arn attribute #10968

Merged
merged 1 commit into from
Nov 21, 2019
Merged

resource/aws_glue_job: Add tags argument and arn attribute #10968

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 99 additions & 49 deletions aws/resource_aws_glue_job.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@ import (
"log"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/service/glue"
"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/helper/validation"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func resourceAwsGlueJob() *schema.Resource {
Expand All @@ -29,6 +31,10 @@ func resourceAwsGlueJob() *schema.Resource {
Deprecated: "Please use attribute `max_capacity' instead. This attribute might be removed in future releases.",
ValidateFunc: validation.IntAtLeast(2),
},
"arn": {
Type: schema.TypeString,
Computed: true,
},
"command": {
Type: schema.TypeList,
Required: true,
Expand Down Expand Up @@ -109,6 +115,7 @@ func resourceAwsGlueJob() *schema.Resource {
Required: true,
ValidateFunc: validateArn,
},
"tags": tagsSchema(),
"timeout": {
Type: schema.TypeInt,
Optional: true,
Expand Down Expand Up @@ -146,6 +153,7 @@ func resourceAwsGlueJobCreate(d *schema.ResourceData, meta interface{}) error {
Command: expandGlueJobCommand(d.Get("command").([]interface{})),
Name: aws.String(name),
Role: aws.String(d.Get("role_arn").(string)),
Tags: keyvaluetags.New(d.Get("tags").(map[string]interface{})).IgnoreAws().GlueTags(),
Timeout: aws.Int64(int64(d.Get("timeout").(int))),
}

Expand Down Expand Up @@ -236,6 +244,15 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error {
return nil
}

jobARN := arn.ARN{
Partition: meta.(*AWSClient).partition,
Service: "glue",
Region: meta.(*AWSClient).region,
AccountID: meta.(*AWSClient).accountid,
Resource: fmt.Sprintf("job/%s", d.Id()),
}.String()
d.Set("arn", jobARN)

if err := d.Set("command", flattenGlueJobCommand(job.Command)); err != nil {
return fmt.Errorf("error setting command: %s", err)
}
Expand All @@ -254,6 +271,17 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error {
d.Set("max_retries", int(aws.Int64Value(job.MaxRetries)))
d.Set("name", job.Name)
d.Set("role_arn", job.Role)

tags, err := keyvaluetags.GlueListTags(conn, jobARN)

if err != nil {
return fmt.Errorf("error listing tags for Glue Job (%s): %s", jobARN, err)
}

if err := d.Set("tags", tags.IgnoreAws().Map()); err != nil {
return fmt.Errorf("error setting tags: %s", err)
}

d.Set("timeout", int(aws.Int64Value(job.Timeout)))
if err := d.Set("security_configuration", job.SecurityConfiguration); err != nil {
return fmt.Errorf("error setting security_configuration: %s", err)
Expand All @@ -271,71 +299,93 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error {
func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*AWSClient).glueconn

jobUpdate := &glue.JobUpdate{
Command: expandGlueJobCommand(d.Get("command").([]interface{})),
Role: aws.String(d.Get("role_arn").(string)),
Timeout: aws.Int64(int64(d.Get("timeout").(int))),
}
if d.HasChange("allocated_capacity") ||
d.HasChange("command") ||
d.HasChange("connections") ||
d.HasChange("default_arguments") ||
d.HasChange("description") ||
d.HasChange("execution_property") ||
d.HasChange("glue_version") ||
d.HasChange("max_capacity") ||
d.HasChange("max_retries") ||
d.HasChange("number_of_workers") ||
d.HasChange("role_arn") ||
d.HasChange("security_configuration") ||
d.HasChange("timeout") ||
d.HasChange("worker_type") {
jobUpdate := &glue.JobUpdate{
Command: expandGlueJobCommand(d.Get("command").([]interface{})),
Role: aws.String(d.Get("role_arn").(string)),
Timeout: aws.Int64(int64(d.Get("timeout").(int))),
}

if v, ok := d.GetOk("number_of_workers"); ok {
jobUpdate.NumberOfWorkers = aws.Int64(int64(v.(int)))
} else {
if v, ok := d.GetOk("max_capacity"); ok {
jobUpdate.MaxCapacity = aws.Float64(v.(float64))
if v, ok := d.GetOk("number_of_workers"); ok {
jobUpdate.NumberOfWorkers = aws.Int64(int64(v.(int)))
} else {
if v, ok := d.GetOk("max_capacity"); ok {
jobUpdate.MaxCapacity = aws.Float64(v.(float64))
}
if d.HasChange("allocated_capacity") {
jobUpdate.MaxCapacity = aws.Float64(float64(d.Get("allocated_capacity").(int)))
log.Printf("[WARN] Using deprecated `allocated_capacity' attribute.")
}
}
if d.HasChange("allocated_capacity") {
jobUpdate.MaxCapacity = aws.Float64(float64(d.Get("allocated_capacity").(int)))
log.Printf("[WARN] Using deprecated `allocated_capacity' attribute.")

if v, ok := d.GetOk("connections"); ok {
jobUpdate.Connections = &glue.ConnectionsList{
Connections: expandStringList(v.([]interface{})),
}
}
}

if v, ok := d.GetOk("connections"); ok {
jobUpdate.Connections = &glue.ConnectionsList{
Connections: expandStringList(v.([]interface{})),
if kv, ok := d.GetOk("default_arguments"); ok {
defaultArgumentsMap := make(map[string]string)
for k, v := range kv.(map[string]interface{}) {
defaultArgumentsMap[k] = v.(string)
}
jobUpdate.DefaultArguments = aws.StringMap(defaultArgumentsMap)
}
}

if kv, ok := d.GetOk("default_arguments"); ok {
defaultArgumentsMap := make(map[string]string)
for k, v := range kv.(map[string]interface{}) {
defaultArgumentsMap[k] = v.(string)
if v, ok := d.GetOk("description"); ok {
jobUpdate.Description = aws.String(v.(string))
}
jobUpdate.DefaultArguments = aws.StringMap(defaultArgumentsMap)
}

if v, ok := d.GetOk("description"); ok {
jobUpdate.Description = aws.String(v.(string))
}
if v, ok := d.GetOk("glue_version"); ok {
jobUpdate.GlueVersion = aws.String(v.(string))
}

if v, ok := d.GetOk("glue_version"); ok {
jobUpdate.GlueVersion = aws.String(v.(string))
}
if v, ok := d.GetOk("execution_property"); ok {
jobUpdate.ExecutionProperty = expandGlueExecutionProperty(v.([]interface{}))
}

if v, ok := d.GetOk("execution_property"); ok {
jobUpdate.ExecutionProperty = expandGlueExecutionProperty(v.([]interface{}))
}
if v, ok := d.GetOk("max_retries"); ok {
jobUpdate.MaxRetries = aws.Int64(int64(v.(int)))
}

if v, ok := d.GetOk("max_retries"); ok {
jobUpdate.MaxRetries = aws.Int64(int64(v.(int)))
}
if v, ok := d.GetOk("security_configuration"); ok {
jobUpdate.SecurityConfiguration = aws.String(v.(string))
}

if v, ok := d.GetOk("security_configuration"); ok {
jobUpdate.SecurityConfiguration = aws.String(v.(string))
}
if v, ok := d.GetOk("worker_type"); ok {
jobUpdate.WorkerType = aws.String(v.(string))
}

if v, ok := d.GetOk("worker_type"); ok {
jobUpdate.WorkerType = aws.String(v.(string))
}
input := &glue.UpdateJobInput{
JobName: aws.String(d.Id()),
JobUpdate: jobUpdate,
}

input := &glue.UpdateJobInput{
JobName: aws.String(d.Id()),
JobUpdate: jobUpdate,
log.Printf("[DEBUG] Updating Glue Job: %s", input)
_, err := conn.UpdateJob(input)
if err != nil {
return fmt.Errorf("error updating Glue Job (%s): %s", d.Id(), err)
}
}

log.Printf("[DEBUG] Updating Glue Job: %s", input)
_, err := conn.UpdateJob(input)
if err != nil {
return fmt.Errorf("error updating Glue Job (%s): %s", d.Id(), err)
if d.HasChange("tags") {
o, n := d.GetChange("tags")
if err := keyvaluetags.GlueUpdateTags(conn, d.Get("arn").(string), o, n); err != nil {
return fmt.Errorf("error updating tags: %s", err)
}
}

return resourceAwsGlueJobRead(d, meta)
Expand Down
90 changes: 90 additions & 0 deletions aws/resource_aws_glue_job_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,13 @@ func TestAccAWSGlueJob_Basic(t *testing.T) {
Config: testAccAWSGlueJobConfig_Required(rName),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job),
testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("job/%s", rName)),
resource.TestCheckResourceAttr(resourceName, "command.#", "1"),
resource.TestCheckResourceAttr(resourceName, "command.0.script_location", "testscriptlocation"),
resource.TestCheckResourceAttr(resourceName, "default_arguments.%", "0"),
resource.TestCheckResourceAttr(resourceName, "name", rName),
resource.TestMatchResourceAttr(resourceName, "role_arn", regexp.MustCompile(fmt.Sprintf("^arn:[^:]+:iam::[^:]+:role/%s", rName))),
resource.TestCheckResourceAttr(resourceName, "tags.%", "0"),
resource.TestCheckResourceAttr(resourceName, "timeout", "2880"),
),
},
Expand Down Expand Up @@ -345,6 +347,51 @@ func TestAccAWSGlueJob_MaxRetries(t *testing.T) {
})
}

func TestAccAWSGlueJob_Tags(t *testing.T) {
var job1, job2, job3 glue.Job

rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5))
resourceName := "aws_glue_job.test"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSGlueJobDestroy,
Steps: []resource.TestStep{
{
Config: testAccAWSGlueJobConfigTags1(rName, "key1", "value1"),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job1),
resource.TestCheckResourceAttr(resourceName, "tags.%", "1"),
resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccAWSGlueJobConfigTags2(rName, "key1", "value1updated", "key2", "value2"),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job2),
resource.TestCheckResourceAttr(resourceName, "tags.%", "2"),
resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1updated"),
resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"),
),
},
{
Config: testAccAWSGlueJobConfigTags1(rName, "key2", "value2"),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job3),
resource.TestCheckResourceAttr(resourceName, "tags.%", "1"),
resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"),
),
},
},
})
}

func TestAccAWSGlueJob_Timeout(t *testing.T) {
var job glue.Job

Expand Down Expand Up @@ -798,6 +845,49 @@ resource "aws_glue_job" "test" {
`, testAccAWSGlueJobConfig_Base(rName), rName)
}

func testAccAWSGlueJobConfigTags1(rName, tagKey1, tagValue1 string) string {
return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(`
resource "aws_glue_job" "test" {
name = %[1]q
number_of_workers = 2
role_arn = "${aws_iam_role.test.arn}"
worker_type = "Standard"

command {
script_location = "testscriptlocation"
}

tags = {
%[2]q = %[3]q
}

depends_on = ["aws_iam_role_policy_attachment.test"]
}
`, rName, tagKey1, tagValue1)
}

func testAccAWSGlueJobConfigTags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string {
return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(`
resource "aws_glue_job" "test" {
name = %[1]q
number_of_workers = 2
role_arn = "${aws_iam_role.test.arn}"
worker_type = "Standard"

command {
script_location = "testscriptlocation"
}

tags = {
%[2]q = %[3]q
%[4]q = %[5]q
}

depends_on = ["aws_iam_role_policy_attachment.test"]
}
`, rName, tagKey1, tagValue1, tagKey2, tagValue2)
}

func testAccAWSGlueJobConfig_Timeout(rName string, timeout int) string {
return fmt.Sprintf(`
%s
Expand Down
2 changes: 2 additions & 0 deletions website/docs/r/glue_job.html.markdown
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ be removed in future releases, please use `max_capacity` instead.
* `max_retries` – (Optional) The maximum number of times to retry this job if it fails.
* `name` – (Required) The name you assign to this job. It must be unique in your account.
* `role_arn` – (Required) The ARN of the IAM role associated with this job.
* `tags` - (Optional) Key-value mapping of resource tags
* `timeout` – (Optional) The job timeout in minutes. The default is 2880 minutes (48 hours).
* `security_configuration` - (Optional) The name of the Security Configuration to be associated with the job.
* `worker_type` - (Optional) The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, or G.2X.
Expand All @@ -79,6 +80,7 @@ be removed in future releases, please use `max_capacity` instead.

In addition to all arguments above, the following attributes are exported:

* `arn` - Amazon Resource Name (ARN) of Glue Job
* `id` - Job name

## Import
Expand Down