From 6a79b7c79186af21422d20509a51c595087d80a4 Mon Sep 17 00:00:00 2001 From: Max Shepard Date: Thu, 17 Aug 2017 18:56:24 -0400 Subject: [PATCH 1/2] Added dynamodb_table_item resource #517 --- aws/provider.go | 1 + aws/resource_aws_dynamodb_table_item.go | 464 ++++++++++++++++++++++++ 2 files changed, 465 insertions(+) create mode 100644 aws/resource_aws_dynamodb_table_item.go diff --git a/aws/provider.go b/aws/provider.go index 28e1b9837fd..bf3e513131b 100644 --- a/aws/provider.go +++ b/aws/provider.go @@ -332,6 +332,7 @@ func Provider() terraform.ResourceProvider { "aws_dx_connection": resourceAwsDxConnection(), "aws_dx_connection_association": resourceAwsDxConnectionAssociation(), "aws_dynamodb_table": resourceAwsDynamoDbTable(), + "aws_dynamodb_table_item": resourceAwsDynamoDbTableItem(), "aws_dynamodb_global_table": resourceAwsDynamoDbGlobalTable(), "aws_ebs_snapshot": resourceAwsEbsSnapshot(), "aws_ebs_volume": resourceAwsEbsVolume(), diff --git a/aws/resource_aws_dynamodb_table_item.go b/aws/resource_aws_dynamodb_table_item.go new file mode 100644 index 00000000000..d8cf8139d42 --- /dev/null +++ b/aws/resource_aws_dynamodb_table_item.go @@ -0,0 +1,464 @@ +package aws + +import ( + "fmt" + "log" + strings "strings" + "time" + + "github.com/hashicorp/terraform/helper/schema" + + "bytes" + "encoding/json" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/dynamodb" + reflect "reflect" +) + +// A number of these are marked as computed because if you don't +// provide a value, DynamoDB will provide you with defaults (which are the +// default values specified below) +func resourceAwsDynamoDbTableItem() *schema.Resource { + return &schema.Resource{ + Create: resourceAwsDynamoDbTableItemCreate, + Read: resourceAwsDynamoDbTableItemRead, + Update: resourceAwsDynamoDbTableItemUpdate, + Delete: resourceAwsDynamoDbTableItemDelete, + Schema: map[string]*schema.Schema{ + "table_name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "hash_key": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "item": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "range_key": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + }, + "query_key": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "range_value": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "hash_value": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "consumed_capacity": &schema.Schema{ + Type: schema.TypeFloat, + Computed: true, + }, + "last_modified": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + }, + } +} + +func resourceAwsDynamoDbTableItemCreate(d *schema.ResourceData, meta interface{}) error { + dynamodbconn := meta.(*AWSClient).dynamodbconn + + tableName := d.Get("table_name").(string) + hashKey := d.Get("hash_key").(string) + rangeKey := d.Get("range_key").(string) + + log.Printf("[DEBUG] DynamoDB item create: %s", tableName) + + item := d.Get("item").(string) + + var av map[string]*dynamodb.AttributeValue + + avDec := json.NewDecoder(strings.NewReader(item)) + + if err := avDec.Decode(&av); err != nil { + return fmt.Errorf("Error deserializing DynamoDB item JSON: %s", err) + } + + exists := false + req := &dynamodb.PutItemInput{ + Item: av, + Expected: map[string]*dynamodb.ExpectedAttributeValue{ + hashKey: { + // Explode if item exists. We didn't create it. + Exists: &exists, + }, + }, + TableName: &tableName, + } + + id := getId(tableName, hashKey, rangeKey, av) + err := retryLoop(func() error { + _, err := dynamodbconn.PutItem(req) + + return err + }, fmt.Sprintf("creating DynamoDB table item '%s'", id)) + + if err != nil { + return err + } + + setQueryKey(d, av, item, hashKey, rangeKey) + + d.SetId(id) + + return resourceAwsDynamoDbTableItemRead(d, meta) +} + +func getId(tableName string, hashKey string, rangeKey string, av map[string]*dynamodb.AttributeValue) string { + hashVal := av[hashKey] + + id := []string{ + tableName, + hashKey, + base64Encode(hashVal.B), + } + + if hashVal.S != nil { + id = append(id, *hashVal.S) + } else { + id = append(id, "") + } + if hashVal.N != nil { + id = append(id, *hashVal.N) + } else { + id = append(id, "") + } + if rangeKey != "" { + rangeVal := av[rangeKey] + + id = append(id, + rangeKey, + base64Encode(rangeVal.B), + ) + + if rangeVal.S != nil { + id = append(id, *rangeVal.S) + } else { + id = append(id, "") + } + + if rangeVal.N != nil { + id = append(id, *rangeVal.N) + } else { + id = append(id, "") + } + + } + + return strings.Join(id, "|") +} + +func resourceAwsDynamoDbTableItemUpdate(d *schema.ResourceData, meta interface{}) error { + log.Printf("[DEBUG] Updating DynamoDB table %s", d.Id()) + dynamodbconn := meta.(*AWSClient).dynamodbconn + + if d.HasChange("item") { + o, n := d.GetChange("item") + + tableName := d.Get("table_name").(string) + hashKey := d.Get("hash_key").(string) + rangeKey := d.Get("range_key").(string) + + newJson := n.(string) + + var newItem map[string]*dynamodb.AttributeValue + + newDec := json.NewDecoder(strings.NewReader(newJson)) + if err := newDec.Decode(&newItem); err != nil { + return fmt.Errorf("Error deserializing DynamoDB item JSON: %s", err) + } + + newQueryKey := getQueryKey(newItem, hashKey, rangeKey) + + updates := map[string]*dynamodb.AttributeValueUpdate{} + + for k, v := range newItem { + // We shouldn't update the key values + skip := false + for qk := range newQueryKey { + if skip = (qk == k); skip { + break + } + } + if skip { + continue + } + + action := "PUT" + updates[k] = &dynamodb.AttributeValueUpdate{ + Action: &action, + Value: v, + } + } + + req := dynamodb.UpdateItemInput{ + AttributeUpdates: updates, + TableName: &tableName, + Key: newQueryKey, + } + + err := retryLoop(func() error { + _, err := dynamodbconn.UpdateItem(&req) + + return err + }, "updating DynamoDB table item '%s'") + + if err != nil { + return err + } + + // If we finished successfully, delete the old record if the query key is different + oldJson := o.(string) + + var oldItem map[string]*dynamodb.AttributeValue + + oldDec := json.NewDecoder(strings.NewReader(oldJson)) + if err := oldDec.Decode(&oldItem); err != nil { + return fmt.Errorf("Error deserializing DynamoDB item JSON: %s", err) + } + + oldQueryKey := getQueryKey(oldItem, hashKey, rangeKey) + + id := getId(tableName, hashKey, rangeKey, newItem) + + if !reflect.DeepEqual(oldQueryKey, newQueryKey) { + req := dynamodb.DeleteItemInput{ + Key: oldQueryKey, + TableName: &tableName, + } + + err := retryLoop(func() error { + _, err := dynamodbconn.DeleteItem(&req) + return err + }, fmt.Sprintf("deleting old DynamoDB item '%s'", id)) + + if err != nil { + return err + } + } + + setQueryKey(d, newItem, newJson, hashKey, rangeKey) + + d.SetId(id) + } + + return resourceAwsDynamoDbTableItemRead(d, meta) +} + +func getQueryKey(av map[string]*dynamodb.AttributeValue, hashKey string, rangeKey string) map[string]*dynamodb.AttributeValue { + qk := map[string]*dynamodb.AttributeValue{} + + flen := 1 + if rangeKey != "" { + flen = 2 + } + + for k, v := range av { + if k == hashKey || k == rangeKey { + qk[k] = v + } + if len(qk) == flen { + break + } + } + + return qk +} + +func setQueryKey(d *schema.ResourceData, av map[string]*dynamodb.AttributeValue, item string, hashKey string, rangeKey string) { + var itemRaw map[string]json.RawMessage + + hashDec := json.NewDecoder(strings.NewReader(item)) + hashDec.Decode(&itemRaw) + + keyRaw := map[string]json.RawMessage{} + + hashRaw := itemRaw[hashKey] + + keyRaw[hashKey] = hashRaw + + hashBytes, _ := hashRaw.MarshalJSON() + d.Set("hash_value", string(hashBytes)) + + if rangeKey != "" { + rangeRaw := itemRaw[rangeKey] + + keyRaw[rangeKey] = rangeRaw + + rangeBytes, _ := rangeRaw.MarshalJSON() + d.Set("range_value", string(rangeBytes)) + } + + queryKeyBuf := bytes.NewBufferString("") + queryKeyEnc := json.NewEncoder(queryKeyBuf) + queryKeyEnc.Encode(keyRaw) + + d.Set("query_key", queryKeyBuf.String()) +} + +func retryLoop(action func() error, actionDetails string) error { + attemptCount := 1 + for attemptCount <= DYNAMODB_MAX_THROTTLE_RETRIES { + err := action() + + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + switch code := awsErr.Code(); code { + case "ThrottlingException": + log.Printf("[DEBUG] Attempt %d/%d: Sleeping for a bit to throttle back %s", attemptCount, DYNAMODB_MAX_THROTTLE_RETRIES, actionDetails) + time.Sleep(DYNAMODB_THROTTLE_SLEEP) + attemptCount += 1 + case "LimitExceededException": + // If we're at resource capacity, error out without retry + if strings.Contains(awsErr.Message(), "Subscriber limit exceeded:") { + return fmt.Errorf("AWS Error %s: %s", actionDetails, err) + } + log.Printf("[DEBUG] Limit on concurrency of %s, sleeping for a bit", actionDetails) + time.Sleep(DYNAMODB_LIMIT_EXCEEDED_SLEEP) + attemptCount += 1 + default: + // Some other non-retryable exception occurred + return fmt.Errorf("AWS Error %s: %s", actionDetails, err) + } + } else { + // Non-AWS exception occurred, give up + return fmt.Errorf("Error %s: %s", actionDetails, err) + } + } else { + return nil + } + } + + // Too many throttling events occurred, give up + return fmt.Errorf("Failed %s after %d attempts", actionDetails, attemptCount) +} + +func resourceAwsDynamoDbTableItemRead(d *schema.ResourceData, meta interface{}) error { + dynamodbconn := meta.(*AWSClient).dynamodbconn + log.Printf("[DEBUG] Loading data for DynamoDB table item '%s'", d.Id()) + + tableName := d.Get("table_name").(string) + + // The record exists, now test if it differs from what is desired + item := d.Get("item").(string) + hashKey := d.Get("hash_key").(string) + rangeKey := d.Get("range_key").(string) + + var av map[string]*dynamodb.AttributeValue + itemDec := json.NewDecoder(strings.NewReader(item)) + itemDec.Decode(&av) + + itemAttributes := []string{} + for k := range av { + itemAttributes = append(itemAttributes, k) + } + + queryKey := getQueryKey(av, hashKey, rangeKey) + + expressionAttributeNames := map[string]*string{} + projection := "#a_" + strings.Join(itemAttributes, ", #a_") + + for _, v := range itemAttributes { + w := v + expressionAttributeNames["#a_"+v] = &w + } + + req := dynamodb.GetItemInput{ + TableName: &tableName, + Key: queryKey, + ProjectionExpression: &projection, + ExpressionAttributeNames: expressionAttributeNames, + } + + result, err := dynamodbconn.GetItem(&req) + + if err != nil { + if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "ResourceNotFoundException" { + log.Printf("[WARN] Dynamodb Table Item (%s) not found, error code (404)", d.Id()) + d.SetId("") + return nil + } + + return fmt.Errorf("Error retrieving DynamoDB table item: %s %s", err, req) + } + + // The record exists, now test if it differs from what is desired + if result.Item != nil && !reflect.DeepEqual(result.Item, av) { + buf := bytes.NewBufferString("") + enc := json.NewEncoder(buf) + enc.Encode(result.Item) + + var itemRaw map[string]map[string]interface{} + + // Reserialize so we get rid of the nulls + dec := json.NewDecoder(strings.NewReader(buf.String())) + dec.Decode(&itemRaw) + + for _, val := range itemRaw { + for typeName, typeVal := range val { + if typeVal == nil { + delete(val, typeName) + } + } + } + + rawBuf := bytes.NewBufferString("") + rawEnc := json.NewEncoder(rawBuf) + rawEnc.Encode(itemRaw) + + d.Set("item", rawBuf.String()) + + id := getId(tableName, hashKey, rangeKey, result.Item) + d.SetId(id) + } else if result.Item == nil { + d.SetId("") + } + + d.Set("consumed_capacity", result.ConsumedCapacity) + + return nil +} + +func resourceAwsDynamoDbTableItemDelete(d *schema.ResourceData, meta interface{}) error { + dynamodbconn := meta.(*AWSClient).dynamodbconn + + tableName := d.Get("table_name").(string) + + item := d.Get("item").(string) + hashKey := d.Get("hash_key").(string) + rangeKey := d.Get("range_key").(string) + + var av map[string]*dynamodb.AttributeValue + itemDec := json.NewDecoder(strings.NewReader(item)) + itemDec.Decode(&av) + + queryKey := getQueryKey(av, hashKey, rangeKey) + + req := dynamodb.DeleteItemInput{ + Key: queryKey, + TableName: &tableName, + } + + err := retryLoop(func() error { + _, err := dynamodbconn.DeleteItem(&req) + + return err + }, fmt.Sprintf("deleting DynamoDB table item '%s'", d.Id())) + + if err != nil { + return err + } + + d.SetId("") + return nil +} From 1c0871b05d8943705f28757a61164f87cd56a268 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Thu, 1 Feb 2018 14:55:27 +0000 Subject: [PATCH 2/2] resource/aws_dynamodb_table_item: Cleanup + add missing bits - Use standard retry helpers - Simplify code by extracting code into reusable functions - Add validation - Add missing acceptance tests - Add missing docs --- aws/config.go | 10 + aws/resource_aws_dynamodb_table_item.go | 501 ++++++------------ aws/resource_aws_dynamodb_table_item_test.go | 363 +++++++++++++ aws/structure.go | 48 ++ website/aws.erb | 4 + .../docs/r/dynamodb_table_item.html.markdown | 62 +++ 6 files changed, 653 insertions(+), 335 deletions(-) create mode 100644 aws/resource_aws_dynamodb_table_item_test.go create mode 100644 website/docs/r/dynamodb_table_item.html.markdown diff --git a/aws/config.go b/aws/config.go index d87df253367..fa37b7f5173 100644 --- a/aws/config.go +++ b/aws/config.go @@ -487,6 +487,16 @@ func (c *Config) Client() (interface{}, error) { } }) + // See https://github.com/aws/aws-sdk-go/pull/1276 + client.dynamodbconn.Handlers.Retry.PushBack(func(r *request.Request) { + if r.Operation.Name != "PutItem" && r.Operation.Name != "UpdateItem" && r.Operation.Name != "DeleteItem" { + return + } + if isAWSErr(r.Error, dynamodb.ErrCodeLimitExceededException, "Subscriber limit exceeded:") { + r.Retryable = aws.Bool(true) + } + }) + return &client, nil } diff --git a/aws/resource_aws_dynamodb_table_item.go b/aws/resource_aws_dynamodb_table_item.go index d8cf8139d42..764bf8b12d9 100644 --- a/aws/resource_aws_dynamodb_table_item.go +++ b/aws/resource_aws_dynamodb_table_item.go @@ -3,462 +3,293 @@ package aws import ( "fmt" "log" - strings "strings" - "time" + "reflect" + "strings" - "github.com/hashicorp/terraform/helper/schema" - - "bytes" - "encoding/json" - "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/dynamodb" - reflect "reflect" + "github.com/hashicorp/terraform/helper/schema" ) -// A number of these are marked as computed because if you don't -// provide a value, DynamoDB will provide you with defaults (which are the -// default values specified below) func resourceAwsDynamoDbTableItem() *schema.Resource { return &schema.Resource{ Create: resourceAwsDynamoDbTableItemCreate, Read: resourceAwsDynamoDbTableItemRead, Update: resourceAwsDynamoDbTableItemUpdate, Delete: resourceAwsDynamoDbTableItemDelete, + Schema: map[string]*schema.Schema{ - "table_name": &schema.Schema{ - Type: schema.TypeString, - Required: true, - }, - "hash_key": &schema.Schema{ + "table_name": { Type: schema.TypeString, Required: true, + ForceNew: true, }, - "item": &schema.Schema{ + "hash_key": { Type: schema.TypeString, Required: true, + ForceNew: true, }, - "range_key": &schema.Schema{ + "range_key": { Type: schema.TypeString, + ForceNew: true, Optional: true, }, - "query_key": &schema.Schema{ - Type: schema.TypeString, - Computed: true, - }, - "range_value": &schema.Schema{ - Type: schema.TypeString, - Computed: true, - }, - "hash_value": &schema.Schema{ - Type: schema.TypeString, - Computed: true, - }, - "consumed_capacity": &schema.Schema{ - Type: schema.TypeFloat, - Computed: true, - }, - "last_modified": &schema.Schema{ - Type: schema.TypeString, - Computed: true, + "item": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validateDynamoDbTableItem, }, }, } } +func validateDynamoDbTableItem(v interface{}, k string) (ws []string, errors []error) { + _, err := expandDynamoDbTableItemAttributes(v.(string)) + if err != nil { + errors = append(errors, fmt.Errorf("Invalid format of %q: %s", k, err)) + } + return +} + func resourceAwsDynamoDbTableItemCreate(d *schema.ResourceData, meta interface{}) error { - dynamodbconn := meta.(*AWSClient).dynamodbconn + conn := meta.(*AWSClient).dynamodbconn tableName := d.Get("table_name").(string) hashKey := d.Get("hash_key").(string) - rangeKey := d.Get("range_key").(string) - - log.Printf("[DEBUG] DynamoDB item create: %s", tableName) - item := d.Get("item").(string) - - var av map[string]*dynamodb.AttributeValue - - avDec := json.NewDecoder(strings.NewReader(item)) - - if err := avDec.Decode(&av); err != nil { - return fmt.Errorf("Error deserializing DynamoDB item JSON: %s", err) + attributes, err := expandDynamoDbTableItemAttributes(item) + if err != nil { + return err } - exists := false - req := &dynamodb.PutItemInput{ - Item: av, + log.Printf("[DEBUG] DynamoDB item create: %s", tableName) + + _, err = conn.PutItem(&dynamodb.PutItemInput{ + Item: attributes, + // Explode if item exists. We didn't create it. Expected: map[string]*dynamodb.ExpectedAttributeValue{ hashKey: { - // Explode if item exists. We didn't create it. - Exists: &exists, + Exists: aws.Bool(false), }, }, - TableName: &tableName, - } - - id := getId(tableName, hashKey, rangeKey, av) - err := retryLoop(func() error { - _, err := dynamodbconn.PutItem(req) - - return err - }, fmt.Sprintf("creating DynamoDB table item '%s'", id)) - + TableName: aws.String(tableName), + }) if err != nil { return err } - setQueryKey(d, av, item, hashKey, rangeKey) + rangeKey := d.Get("range_key").(string) + id := buildDynamoDbTableItemId(tableName, hashKey, rangeKey, attributes) d.SetId(id) return resourceAwsDynamoDbTableItemRead(d, meta) } -func getId(tableName string, hashKey string, rangeKey string, av map[string]*dynamodb.AttributeValue) string { - hashVal := av[hashKey] - - id := []string{ - tableName, - hashKey, - base64Encode(hashVal.B), - } - - if hashVal.S != nil { - id = append(id, *hashVal.S) - } else { - id = append(id, "") - } - if hashVal.N != nil { - id = append(id, *hashVal.N) - } else { - id = append(id, "") - } - if rangeKey != "" { - rangeVal := av[rangeKey] - - id = append(id, - rangeKey, - base64Encode(rangeVal.B), - ) - - if rangeVal.S != nil { - id = append(id, *rangeVal.S) - } else { - id = append(id, "") - } - - if rangeVal.N != nil { - id = append(id, *rangeVal.N) - } else { - id = append(id, "") - } - - } - - return strings.Join(id, "|") -} - func resourceAwsDynamoDbTableItemUpdate(d *schema.ResourceData, meta interface{}) error { log.Printf("[DEBUG] Updating DynamoDB table %s", d.Id()) - dynamodbconn := meta.(*AWSClient).dynamodbconn + conn := meta.(*AWSClient).dynamodbconn if d.HasChange("item") { - o, n := d.GetChange("item") - tableName := d.Get("table_name").(string) hashKey := d.Get("hash_key").(string) rangeKey := d.Get("range_key").(string) - newJson := n.(string) + oldItem, newItem := d.GetChange("item") - var newItem map[string]*dynamodb.AttributeValue - - newDec := json.NewDecoder(strings.NewReader(newJson)) - if err := newDec.Decode(&newItem); err != nil { - return fmt.Errorf("Error deserializing DynamoDB item JSON: %s", err) + attributes, err := expandDynamoDbTableItemAttributes(newItem.(string)) + if err != nil { + return err } - - newQueryKey := getQueryKey(newItem, hashKey, rangeKey) + newQueryKey := buildDynamoDbTableItemQueryKey(attributes, hashKey, rangeKey) updates := map[string]*dynamodb.AttributeValueUpdate{} - - for k, v := range newItem { - // We shouldn't update the key values - skip := false - for qk := range newQueryKey { - if skip = (qk == k); skip { - break - } - } - if skip { + for key, value := range attributes { + // Hash keys are not updatable, so we'll basically create + // a new record and delete the old one below + if key == hashKey { continue } - - action := "PUT" - updates[k] = &dynamodb.AttributeValueUpdate{ - Action: &action, - Value: v, + updates[key] = &dynamodb.AttributeValueUpdate{ + Action: aws.String(dynamodb.AttributeActionPut), + Value: value, } } - req := dynamodb.UpdateItemInput{ + _, err = conn.UpdateItem(&dynamodb.UpdateItemInput{ AttributeUpdates: updates, - TableName: &tableName, + TableName: aws.String(tableName), Key: newQueryKey, - } - - err := retryLoop(func() error { - _, err := dynamodbconn.UpdateItem(&req) - - return err - }, "updating DynamoDB table item '%s'") - + }) if err != nil { return err } - // If we finished successfully, delete the old record if the query key is different - oldJson := o.(string) - - var oldItem map[string]*dynamodb.AttributeValue - - oldDec := json.NewDecoder(strings.NewReader(oldJson)) - if err := oldDec.Decode(&oldItem); err != nil { - return fmt.Errorf("Error deserializing DynamoDB item JSON: %s", err) + oItem := oldItem.(string) + oldAttributes, err := expandDynamoDbTableItemAttributes(oItem) + if err != nil { + return err } - oldQueryKey := getQueryKey(oldItem, hashKey, rangeKey) - - id := getId(tableName, hashKey, rangeKey, newItem) - + // New record is created via UpdateItem in case we're changing hash key + // so we need to get rid of the old one + oldQueryKey := buildDynamoDbTableItemQueryKey(oldAttributes, hashKey, rangeKey) if !reflect.DeepEqual(oldQueryKey, newQueryKey) { - req := dynamodb.DeleteItemInput{ + log.Printf("[DEBUG] Deleting old record: %#v", oldQueryKey) + _, err := conn.DeleteItem(&dynamodb.DeleteItemInput{ Key: oldQueryKey, - TableName: &tableName, - } - - err := retryLoop(func() error { - _, err := dynamodbconn.DeleteItem(&req) - return err - }, fmt.Sprintf("deleting old DynamoDB item '%s'", id)) - + TableName: aws.String(tableName), + }) if err != nil { return err } } - setQueryKey(d, newItem, newJson, hashKey, rangeKey) - + id := buildDynamoDbTableItemId(tableName, hashKey, rangeKey, attributes) d.SetId(id) } return resourceAwsDynamoDbTableItemRead(d, meta) } -func getQueryKey(av map[string]*dynamodb.AttributeValue, hashKey string, rangeKey string) map[string]*dynamodb.AttributeValue { - qk := map[string]*dynamodb.AttributeValue{} +func resourceAwsDynamoDbTableItemRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).dynamodbconn - flen := 1 - if rangeKey != "" { - flen = 2 - } + log.Printf("[DEBUG] Loading data for DynamoDB table item '%s'", d.Id()) - for k, v := range av { - if k == hashKey || k == rangeKey { - qk[k] = v - } - if len(qk) == flen { - break - } + tableName := d.Get("table_name").(string) + hashKey := d.Get("hash_key").(string) + rangeKey := d.Get("range_key").(string) + attributes, err := expandDynamoDbTableItemAttributes(d.Get("item").(string)) + if err != nil { + return err } - return qk -} - -func setQueryKey(d *schema.ResourceData, av map[string]*dynamodb.AttributeValue, item string, hashKey string, rangeKey string) { - var itemRaw map[string]json.RawMessage - - hashDec := json.NewDecoder(strings.NewReader(item)) - hashDec.Decode(&itemRaw) - - keyRaw := map[string]json.RawMessage{} - - hashRaw := itemRaw[hashKey] - - keyRaw[hashKey] = hashRaw - - hashBytes, _ := hashRaw.MarshalJSON() - d.Set("hash_value", string(hashBytes)) - - if rangeKey != "" { - rangeRaw := itemRaw[rangeKey] - - keyRaw[rangeKey] = rangeRaw + result, err := conn.GetItem(&dynamodb.GetItemInput{ + TableName: aws.String(tableName), + ConsistentRead: aws.Bool(true), + Key: buildDynamoDbTableItemQueryKey(attributes, hashKey, rangeKey), + ProjectionExpression: buildDynamoDbProjectionExpression(attributes), + ExpressionAttributeNames: buildDynamoDbExpressionAttributeNames(attributes), + }) + if err != nil { + if isAWSErr(err, dynamodb.ErrCodeResourceNotFoundException, "") { + log.Printf("[WARN] Dynamodb Table Item (%s) not found, error code (404)", d.Id()) + d.SetId("") + return nil + } - rangeBytes, _ := rangeRaw.MarshalJSON() - d.Set("range_value", string(rangeBytes)) + return fmt.Errorf("Error retrieving DynamoDB table item: %s", err) } - queryKeyBuf := bytes.NewBufferString("") - queryKeyEnc := json.NewEncoder(queryKeyBuf) - queryKeyEnc.Encode(keyRaw) - - d.Set("query_key", queryKeyBuf.String()) -} - -func retryLoop(action func() error, actionDetails string) error { - attemptCount := 1 - for attemptCount <= DYNAMODB_MAX_THROTTLE_RETRIES { - err := action() + if result.Item == nil { + log.Printf("[WARN] Dynamodb Table Item (%s) not found", d.Id()) + d.SetId("") + return nil + } + // The record exists, now test if it differs from what is desired + if !reflect.DeepEqual(result.Item, attributes) { + itemAttrs, err := flattenDynamoDbTableItemAttributes(result.Item) if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - switch code := awsErr.Code(); code { - case "ThrottlingException": - log.Printf("[DEBUG] Attempt %d/%d: Sleeping for a bit to throttle back %s", attemptCount, DYNAMODB_MAX_THROTTLE_RETRIES, actionDetails) - time.Sleep(DYNAMODB_THROTTLE_SLEEP) - attemptCount += 1 - case "LimitExceededException": - // If we're at resource capacity, error out without retry - if strings.Contains(awsErr.Message(), "Subscriber limit exceeded:") { - return fmt.Errorf("AWS Error %s: %s", actionDetails, err) - } - log.Printf("[DEBUG] Limit on concurrency of %s, sleeping for a bit", actionDetails) - time.Sleep(DYNAMODB_LIMIT_EXCEEDED_SLEEP) - attemptCount += 1 - default: - // Some other non-retryable exception occurred - return fmt.Errorf("AWS Error %s: %s", actionDetails, err) - } - } else { - // Non-AWS exception occurred, give up - return fmt.Errorf("Error %s: %s", actionDetails, err) - } - } else { - return nil + return err } + d.Set("item", itemAttrs) + id := buildDynamoDbTableItemId(tableName, hashKey, rangeKey, result.Item) + d.SetId(id) } - // Too many throttling events occurred, give up - return fmt.Errorf("Failed %s after %d attempts", actionDetails, attemptCount) + return nil } -func resourceAwsDynamoDbTableItemRead(d *schema.ResourceData, meta interface{}) error { - dynamodbconn := meta.(*AWSClient).dynamodbconn - log.Printf("[DEBUG] Loading data for DynamoDB table item '%s'", d.Id()) - - tableName := d.Get("table_name").(string) +func resourceAwsDynamoDbTableItemDelete(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).dynamodbconn - // The record exists, now test if it differs from what is desired - item := d.Get("item").(string) + attributes, err := expandDynamoDbTableItemAttributes(d.Get("item").(string)) + if err != nil { + return err + } hashKey := d.Get("hash_key").(string) rangeKey := d.Get("range_key").(string) + queryKey := buildDynamoDbTableItemQueryKey(attributes, hashKey, rangeKey) - var av map[string]*dynamodb.AttributeValue - itemDec := json.NewDecoder(strings.NewReader(item)) - itemDec.Decode(&av) + _, err = conn.DeleteItem(&dynamodb.DeleteItemInput{ + Key: queryKey, + TableName: aws.String(d.Get("table_name").(string)), + }) + return err +} - itemAttributes := []string{} - for k := range av { - itemAttributes = append(itemAttributes, k) +// Helpers + +func buildDynamoDbExpressionAttributeNames(attrs map[string]*dynamodb.AttributeValue) map[string]*string { + names := map[string]*string{} + for key, _ := range attrs { + names["#a_"+key] = aws.String(key) } - queryKey := getQueryKey(av, hashKey, rangeKey) + return names +} + +func buildDynamoDbProjectionExpression(attrs map[string]*dynamodb.AttributeValue) *string { + keys := []string{} + for key, _ := range attrs { + keys = append(keys, key) + } + return aws.String("#a_" + strings.Join(keys, ", #a_")) +} - expressionAttributeNames := map[string]*string{} - projection := "#a_" + strings.Join(itemAttributes, ", #a_") +func buildDynamoDbTableItemId(tableName string, hashKey string, rangeKey string, attrs map[string]*dynamodb.AttributeValue) string { + hashVal := attrs[hashKey] - for _, v := range itemAttributes { - w := v - expressionAttributeNames["#a_"+v] = &w + id := []string{ + tableName, + hashKey, + base64Encode(hashVal.B), } - req := dynamodb.GetItemInput{ - TableName: &tableName, - Key: queryKey, - ProjectionExpression: &projection, - ExpressionAttributeNames: expressionAttributeNames, + if hashVal.S != nil { + id = append(id, *hashVal.S) + } else { + id = append(id, "") + } + if hashVal.N != nil { + id = append(id, *hashVal.N) + } else { + id = append(id, "") } + if rangeKey != "" { + rangeVal := attrs[rangeKey] - result, err := dynamodbconn.GetItem(&req) + id = append(id, + rangeKey, + base64Encode(rangeVal.B), + ) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "ResourceNotFoundException" { - log.Printf("[WARN] Dynamodb Table Item (%s) not found, error code (404)", d.Id()) - d.SetId("") - return nil + if rangeVal.S != nil { + id = append(id, *rangeVal.S) + } else { + id = append(id, "") } - return fmt.Errorf("Error retrieving DynamoDB table item: %s %s", err, req) - } - - // The record exists, now test if it differs from what is desired - if result.Item != nil && !reflect.DeepEqual(result.Item, av) { - buf := bytes.NewBufferString("") - enc := json.NewEncoder(buf) - enc.Encode(result.Item) - - var itemRaw map[string]map[string]interface{} - - // Reserialize so we get rid of the nulls - dec := json.NewDecoder(strings.NewReader(buf.String())) - dec.Decode(&itemRaw) - - for _, val := range itemRaw { - for typeName, typeVal := range val { - if typeVal == nil { - delete(val, typeName) - } - } + if rangeVal.N != nil { + id = append(id, *rangeVal.N) + } else { + id = append(id, "") } - rawBuf := bytes.NewBufferString("") - rawEnc := json.NewEncoder(rawBuf) - rawEnc.Encode(itemRaw) - - d.Set("item", rawBuf.String()) - - id := getId(tableName, hashKey, rangeKey, result.Item) - d.SetId(id) - } else if result.Item == nil { - d.SetId("") } - d.Set("consumed_capacity", result.ConsumedCapacity) - - return nil + return strings.Join(id, "|") } -func resourceAwsDynamoDbTableItemDelete(d *schema.ResourceData, meta interface{}) error { - dynamodbconn := meta.(*AWSClient).dynamodbconn - - tableName := d.Get("table_name").(string) - - item := d.Get("item").(string) - hashKey := d.Get("hash_key").(string) - rangeKey := d.Get("range_key").(string) - - var av map[string]*dynamodb.AttributeValue - itemDec := json.NewDecoder(strings.NewReader(item)) - itemDec.Decode(&av) - - queryKey := getQueryKey(av, hashKey, rangeKey) - - req := dynamodb.DeleteItemInput{ - Key: queryKey, - TableName: &tableName, +func buildDynamoDbTableItemQueryKey(attrs map[string]*dynamodb.AttributeValue, hashKey string, rangeKey string) map[string]*dynamodb.AttributeValue { + queryKey := map[string]*dynamodb.AttributeValue{ + hashKey: attrs[hashKey], } - - err := retryLoop(func() error { - _, err := dynamodbconn.DeleteItem(&req) - - return err - }, fmt.Sprintf("deleting DynamoDB table item '%s'", d.Id())) - - if err != nil { - return err + if rangeKey != "" { + queryKey[rangeKey] = attrs[rangeKey] } - d.SetId("") - return nil + return queryKey } diff --git a/aws/resource_aws_dynamodb_table_item_test.go b/aws/resource_aws_dynamodb_table_item_test.go new file mode 100644 index 00000000000..d18c2b7771e --- /dev/null +++ b/aws/resource_aws_dynamodb_table_item_test.go @@ -0,0 +1,363 @@ +package aws + +import ( + "fmt" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/hashicorp/terraform/helper/acctest" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +func TestAccAWSDynamoDbTableItem_basic(t *testing.T) { + var conf dynamodb.GetItemOutput + + tableName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(8)) + hashKey := "hashKey" + itemContent := `{ + "hashKey": {"S": "something"}, + "one": {"N": "11111"}, + "two": {"N": "22222"}, + "three": {"N": "33333"}, + "four": {"N": "44444"} +}` + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSDynamoDbItemDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSDynamoDbItemConfigBasic(tableName, hashKey, itemContent), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSDynamoDbTableItemExists("aws_dynamodb_table_item.test", &conf), + testAccCheckAWSDynamoDbTableItemCount(tableName, 1), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "hash_key", hashKey), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "table_name", tableName), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "item", itemContent+"\n"), + ), + }, + }, + }) +} + +func TestAccAWSDynamoDbTableItem_rangeKey(t *testing.T) { + var conf dynamodb.GetItemOutput + + tableName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(8)) + hashKey := "hashKey" + rangeKey := "rangeKey" + itemContent := `{ + "hashKey": {"S": "something"}, + "rangeKey": {"S": "something-else"}, + "one": {"N": "11111"}, + "two": {"N": "22222"}, + "three": {"N": "33333"}, + "four": {"N": "44444"} +}` + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSDynamoDbItemDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSDynamoDbItemConfigWithRangeKey(tableName, hashKey, rangeKey, itemContent), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSDynamoDbTableItemExists("aws_dynamodb_table_item.test", &conf), + testAccCheckAWSDynamoDbTableItemCount(tableName, 1), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "hash_key", hashKey), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "range_key", rangeKey), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "table_name", tableName), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "item", itemContent+"\n"), + ), + }, + }, + }) +} + +func TestAccAWSDynamoDbTableItem_withMultipleItems(t *testing.T) { + var conf1 dynamodb.GetItemOutput + var conf2 dynamodb.GetItemOutput + + tableName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(8)) + hashKey := "hashKey" + rangeKey := "rangeKey" + firstItem := `{ + "hashKey": {"S": "something"}, + "rangeKey": {"S": "first"}, + "one": {"N": "11111"}, + "two": {"N": "22222"}, + "three": {"N": "33333"} +}` + secondItem := `{ + "hashKey": {"S": "something"}, + "rangeKey": {"S": "second"}, + "one": {"S": "one"}, + "two": {"S": "two"}, + "three": {"S": "three"}, + "four": {"S": "four"} +}` + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSDynamoDbItemDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSDynamoDbItemConfigWithMultipleItems(tableName, hashKey, rangeKey, firstItem, secondItem), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSDynamoDbTableItemExists("aws_dynamodb_table_item.test1", &conf1), + testAccCheckAWSDynamoDbTableItemExists("aws_dynamodb_table_item.test2", &conf2), + testAccCheckAWSDynamoDbTableItemCount(tableName, 2), + + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test1", "hash_key", hashKey), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test1", "range_key", rangeKey), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test1", "table_name", tableName), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test1", "item", firstItem+"\n"), + + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test2", "hash_key", hashKey), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test2", "range_key", rangeKey), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test2", "table_name", tableName), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test2", "item", secondItem+"\n"), + ), + }, + }, + }) +} + +func TestAccAWSDynamoDbTableItem_update(t *testing.T) { + var conf dynamodb.GetItemOutput + + tableName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(8)) + hashKey := "hashKey" + + itemBefore := `{ + "hashKey": {"S": "before"}, + "one": {"N": "11111"}, + "two": {"N": "22222"}, + "three": {"N": "33333"}, + "four": {"N": "44444"} +}` + itemAfter := `{ + "hashKey": {"S": "before"}, + "one": {"N": "11111"}, + "two": {"N": "22222"}, + "new": {"S": "shiny new one"} +}` + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSDynamoDbItemDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSDynamoDbItemConfigBasic(tableName, hashKey, itemBefore), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSDynamoDbTableItemExists("aws_dynamodb_table_item.test", &conf), + testAccCheckAWSDynamoDbTableItemCount(tableName, 1), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "hash_key", hashKey), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "table_name", tableName), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "item", itemBefore+"\n"), + ), + }, + { + Config: testAccAWSDynamoDbItemConfigBasic(tableName, hashKey, itemAfter), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSDynamoDbTableItemExists("aws_dynamodb_table_item.test", &conf), + testAccCheckAWSDynamoDbTableItemCount(tableName, 1), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "hash_key", hashKey), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "table_name", tableName), + resource.TestCheckResourceAttr("aws_dynamodb_table_item.test", "item", itemAfter+"\n"), + ), + }, + }, + }) +} + +func testAccCheckAWSDynamoDbItemDestroy(s *terraform.State) error { + conn := testAccProvider.Meta().(*AWSClient).dynamodbconn + + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_dynamodb_table_item" { + continue + } + + attrs := rs.Primary.Attributes + attributes, err := expandDynamoDbTableItemAttributes(attrs["item"]) + if err != nil { + return err + } + + result, err := conn.GetItem(&dynamodb.GetItemInput{ + TableName: aws.String(attrs["table_name"]), + ConsistentRead: aws.Bool(true), + Key: buildDynamoDbTableItemQueryKey(attributes, attrs["hash_key"], attrs["range_key"]), + ProjectionExpression: buildDynamoDbProjectionExpression(attributes), + ExpressionAttributeNames: buildDynamoDbExpressionAttributeNames(attributes), + }) + if err != nil { + if isAWSErr(err, dynamodb.ErrCodeResourceNotFoundException, "") { + return nil + } + return fmt.Errorf("Error retrieving DynamoDB table item: %s", err) + } + if result.Item == nil { + return nil + } + + return fmt.Errorf("DynamoDB table item %s still exists.", rs.Primary.ID) + } + + return nil +} + +func testAccCheckAWSDynamoDbTableItemExists(n string, item *dynamodb.GetItemOutput) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("No DynamoDB table item ID specified!") + } + + conn := testAccProvider.Meta().(*AWSClient).dynamodbconn + + attrs := rs.Primary.Attributes + attributes, err := expandDynamoDbTableItemAttributes(attrs["item"]) + if err != nil { + return err + } + + result, err := conn.GetItem(&dynamodb.GetItemInput{ + TableName: aws.String(attrs["table_name"]), + ConsistentRead: aws.Bool(true), + Key: buildDynamoDbTableItemQueryKey(attributes, attrs["hash_key"], attrs["range_key"]), + ProjectionExpression: buildDynamoDbProjectionExpression(attributes), + ExpressionAttributeNames: buildDynamoDbExpressionAttributeNames(attributes), + }) + if err != nil { + return fmt.Errorf("[ERROR] Problem getting table item '%s': %s", rs.Primary.ID, err) + } + + *item = *result + + return nil + } +} + +func testAccCheckAWSDynamoDbTableItemCount(tableName string, count int64) resource.TestCheckFunc { + return func(s *terraform.State) error { + conn := testAccProvider.Meta().(*AWSClient).dynamodbconn + out, err := conn.Scan(&dynamodb.ScanInput{ + ConsistentRead: aws.Bool(true), + TableName: aws.String(tableName), + Select: aws.String(dynamodb.SelectCount), + }) + if err != nil { + return err + } + expectedCount := count + if *out.Count != expectedCount { + return fmt.Errorf("Expected %d items, got %d", expectedCount, *out.Count) + } + return nil + } +} + +func testAccAWSDynamoDbItemConfigBasic(tableName, hashKey, item string) string { + return fmt.Sprintf(` +resource "aws_dynamodb_table" "test" { + name = "%s" + read_capacity = 10 + write_capacity = 10 + hash_key = "%s" + + attribute { + name = "%s" + type = "S" + } +} + +resource "aws_dynamodb_table_item" "test" { + table_name = "${aws_dynamodb_table.test.name}" + hash_key = "${aws_dynamodb_table.test.hash_key}" + item = <aws_dynamodb_table + > + aws_dynamodb_table_item + + diff --git a/website/docs/r/dynamodb_table_item.html.markdown b/website/docs/r/dynamodb_table_item.html.markdown new file mode 100644 index 00000000000..8ac534b2c83 --- /dev/null +++ b/website/docs/r/dynamodb_table_item.html.markdown @@ -0,0 +1,62 @@ +--- +layout: "aws" +page_title: "AWS: dynamodb_table_item" +sidebar_current: "docs-aws-resource-dynamodb-table-item" +description: |- + Provides a DynamoDB table item resource +--- + +# aws_dynamodb_table_item + +Provides a DynamoDB table item resource + +-> **Note:** This resource is not meant to be used for managing large amounts of data in your table, it is not designed to scale. + You should perform **regular backups** of all data in the table, see [AWS docs for more](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/BackupRestore.html). + +## Example Usage + +```hcl +resource "aws_dynamodb_table_item" "example" { + table_name = "${aws_dynamodb_table.example.name}" + hash_key = "${aws_dynamodb_table.example.hash_key}" + item = <