From 4df9a47da8aa53fd5f391408d064c4b6ebd6baaf Mon Sep 17 00:00:00 2001 From: Peter Leschev Date: Thu, 7 Sep 2017 11:52:19 +1000 Subject: [PATCH 1/4] First attempt at resolving #1601 --- ...ce_aws_kinesis_firehose_delivery_stream.go | 50 +++++++++++++++++++ ...sis_firehose_delivery_stream.html.markdown | 5 ++ 2 files changed, 55 insertions(+) diff --git a/aws/resource_aws_kinesis_firehose_delivery_stream.go b/aws/resource_aws_kinesis_firehose_delivery_stream.go index 10fce25e5f8..36d0dd08509 100644 --- a/aws/resource_aws_kinesis_firehose_delivery_stream.go +++ b/aws/resource_aws_kinesis_firehose_delivery_stream.go @@ -133,6 +133,28 @@ func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource { }, }, + "source_configuration": { + Type: schema.TypeList, + ForceNew: true, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "kinesis_stream_arn": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validateArn, + }, + + "role_arn": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validateArn, + }, + }, + }, + }, + "destination": { Type: schema.TypeString, Required: true, @@ -445,6 +467,16 @@ func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource { } } +func createSourceConfig(source map[string]interface{}) *firehose.KinesisStreamSourceConfiguration { + + configuration := &firehose.KinesisStreamSourceConfiguration{ + KinesisStreamARN: aws.String(source["kinesis_stream_arn"].(string)), + RoleARN: aws.String(source["role_arn"].(string)), + } + + return configuration +} + func createS3Config(d *schema.ResourceData) *firehose.S3DestinationConfiguration { s3 := d.Get("s3_configuration").([]interface{})[0].(map[string]interface{}) @@ -490,6 +522,16 @@ func createExtendedS3Config(d *schema.ResourceData) *firehose.ExtendedS3Destinat return configuration } +func updateSourceConfig(source map[string]interface{}) *firehose.KinesisStreamSourceConfiguration { + + configuration := &firehose.KinesisStreamSourceConfiguration{ + KinesisStreamARN: aws.String(source["kinesis_stream_arn"].(string)), + RoleARN: aws.String(source["role_arn"].(string)), + } + + return configuration +} + func updateS3Config(d *schema.ResourceData) *firehose.S3DestinationUpdate { s3 := d.Get("s3_configuration").([]interface{})[0].(map[string]interface{}) @@ -804,6 +846,14 @@ func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta DeliveryStreamName: aws.String(sn), } + if v, ok := d.GetOk("source_configuration"); ok { + sourceConfig := createSourceConfig(v.([]interface{})[0].(map[string]interface{})) + createInput.KinesisStreamSourceConfiguration = sourceConfig + createInput.DeliveryStreamType = aws.String(firehose.DeliveryStreamTypeKinesisStreamAsSource) + } else { + createInput.DeliveryStreamType = aws.String(firehose.DeliveryStreamTypeDirectPut) + } + if d.Get("destination").(string) == "extended_s3" { extendedS3Config := createExtendedS3Config(d) createInput.ExtendedS3DestinationConfiguration = extendedS3Config diff --git a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown index 751ac73ec84..86c35911b25 100644 --- a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown +++ b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown @@ -207,6 +207,7 @@ The following arguments are supported: * `name` - (Required) A name to identify the stream. This is unique to the AWS account and region the Stream is created in. +* `source_configuration` - (Optional) Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream. * `destination` – (Required) This is the destination to where the data is delivered. The only options are `s3` (Deprecated, use `extended_s3` instead), `extended_s3`, `redshift`, and `elasticsearch`. * `s3_configuration` - (Optional, Deprecated, see/use `extended_s3_configuration` unless `destination` is `redshift`) Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. @@ -215,6 +216,10 @@ is redshift). More details are given below. Using `redshift_configuration` requires the user to also specify a `s3_configuration` block. More details are given below. +The `source_configuration` object supports the following: +* `kinesis_stream_arn` (Required) The kinesis stream used as the source of the firehose delivery stream. +* `role_arn` (Required) The ARN of the role that provides access to the source Kinesis stream. + The `s3_configuration` object supports the following: * `role_arn` - (Required) The ARN of the AWS credentials. From c69bb33f423758f331b6f037ebbb033a30723de8 Mon Sep 17 00:00:00 2001 From: Peter Leschev Date: Thu, 7 Sep 2017 22:12:01 +1000 Subject: [PATCH 2/4] Removing unused method --- aws/resource_aws_kinesis_firehose_delivery_stream.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/aws/resource_aws_kinesis_firehose_delivery_stream.go b/aws/resource_aws_kinesis_firehose_delivery_stream.go index 36d0dd08509..13e547b2200 100644 --- a/aws/resource_aws_kinesis_firehose_delivery_stream.go +++ b/aws/resource_aws_kinesis_firehose_delivery_stream.go @@ -522,16 +522,6 @@ func createExtendedS3Config(d *schema.ResourceData) *firehose.ExtendedS3Destinat return configuration } -func updateSourceConfig(source map[string]interface{}) *firehose.KinesisStreamSourceConfiguration { - - configuration := &firehose.KinesisStreamSourceConfiguration{ - KinesisStreamARN: aws.String(source["kinesis_stream_arn"].(string)), - RoleARN: aws.String(source["role_arn"].(string)), - } - - return configuration -} - func updateS3Config(d *schema.ResourceData) *firehose.S3DestinationUpdate { s3 := d.Get("s3_configuration").([]interface{})[0].(map[string]interface{}) From 50e7345eb83603cfbaeb85f02d83c138bafd7298 Mon Sep 17 00:00:00 2001 From: Ivan Lopez Date: Tue, 19 Sep 2017 16:16:39 +0200 Subject: [PATCH 3/4] Add acceptance test for aws_kinesis_firehose_delivery_stream using Kinesis Stream as source --- ...s_kinesis_firehose_delivery_stream_test.go | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/aws/resource_aws_kinesis_firehose_delivery_stream_test.go b/aws/resource_aws_kinesis_firehose_delivery_stream_test.go index 070f1e8b9b0..66afcdb5b2c 100644 --- a/aws/resource_aws_kinesis_firehose_delivery_stream_test.go +++ b/aws/resource_aws_kinesis_firehose_delivery_stream_test.go @@ -38,6 +38,28 @@ func TestAccAWSKinesisFirehoseDeliveryStream_s3basic(t *testing.T) { }) } +func TestAccAWSKinesisFirehoseDeliveryStream_s3KinesisStreamSource(t *testing.T) { + var stream firehose.DeliveryStreamDescription + ri := acctest.RandInt() + config := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3KinesisStreamSource, + ri, os.Getenv("AWS_ACCOUNT_ID"), ri, ri, ri, ri, os.Getenv("AWS_ACCOUNT_ID"), ri, ri, ri) + + resource.Test(t, resource.TestCase{ + PreCheck: testAccKinesisFirehosePreCheck(t), + Providers: testAccProviders, + CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy, + Steps: []resource.TestStep{ + { + Config: config, + Check: resource.ComposeTestCheckFunc( + testAccCheckKinesisFirehoseDeliveryStreamExists("aws_kinesis_firehose_delivery_stream.test_stream", &stream), + testAccCheckAWSKinesisFirehoseDeliveryStreamAttributes(&stream, nil, nil, nil, nil), + ), + }, + }, + }) +} + func TestAccAWSKinesisFirehoseDeliveryStream_s3WithCloudwatchLogging(t *testing.T) { var stream firehose.DeliveryStreamDescription ri := acctest.RandInt() @@ -614,6 +636,62 @@ EOF ` +const testAccFirehoseKinesisStreamSource = ` +resource "aws_kinesis_stream" "source" { + name = "terraform-kinesis-source-stream-basictest-%d" + shard_count = 1 +} + +resource "aws_iam_role" "kinesis_source" { + name = "tf_acctest_kinesis_source_role_%d" + assume_role_policy = < Date: Sat, 30 Sep 2017 10:41:20 +1000 Subject: [PATCH 4/4] Updated "source_configuration" to be "kinesis_source_configuration" --- aws/resource_aws_kinesis_firehose_delivery_stream.go | 4 ++-- aws/resource_aws_kinesis_firehose_delivery_stream_test.go | 2 +- website/docs/r/kinesis_firehose_delivery_stream.html.markdown | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aws/resource_aws_kinesis_firehose_delivery_stream.go b/aws/resource_aws_kinesis_firehose_delivery_stream.go index 13e547b2200..7739c10e20f 100644 --- a/aws/resource_aws_kinesis_firehose_delivery_stream.go +++ b/aws/resource_aws_kinesis_firehose_delivery_stream.go @@ -133,7 +133,7 @@ func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource { }, }, - "source_configuration": { + "kinesis_source_configuration": { Type: schema.TypeList, ForceNew: true, Optional: true, @@ -836,7 +836,7 @@ func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta DeliveryStreamName: aws.String(sn), } - if v, ok := d.GetOk("source_configuration"); ok { + if v, ok := d.GetOk("kinesis_source_configuration"); ok { sourceConfig := createSourceConfig(v.([]interface{})[0].(map[string]interface{})) createInput.KinesisStreamSourceConfiguration = sourceConfig createInput.DeliveryStreamType = aws.String(firehose.DeliveryStreamTypeKinesisStreamAsSource) diff --git a/aws/resource_aws_kinesis_firehose_delivery_stream_test.go b/aws/resource_aws_kinesis_firehose_delivery_stream_test.go index 66afcdb5b2c..1e88a1f7e21 100644 --- a/aws/resource_aws_kinesis_firehose_delivery_stream_test.go +++ b/aws/resource_aws_kinesis_firehose_delivery_stream_test.go @@ -801,7 +801,7 @@ var testAccKinesisFirehoseDeliveryStreamConfig_s3KinesisStreamSource = testAccKi resource "aws_kinesis_firehose_delivery_stream" "test_stream" { depends_on = ["aws_iam_role_policy.firehose", "aws_iam_role_policy.kinesis_source"] name = "terraform-kinesis-firehose-basictest-%d" - source_configuration { + kinesis_source_configuration { kinesis_stream_arn = "${aws_kinesis_stream.source.arn}" role_arn = "${aws_iam_role.kinesis_source.arn}" } diff --git a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown index 86c35911b25..09a52589ad0 100644 --- a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown +++ b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown @@ -207,7 +207,7 @@ The following arguments are supported: * `name` - (Required) A name to identify the stream. This is unique to the AWS account and region the Stream is created in. -* `source_configuration` - (Optional) Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream. +* `kinesis_source_configuration` - (Optional) Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream. * `destination` – (Required) This is the destination to where the data is delivered. The only options are `s3` (Deprecated, use `extended_s3` instead), `extended_s3`, `redshift`, and `elasticsearch`. * `s3_configuration` - (Optional, Deprecated, see/use `extended_s3_configuration` unless `destination` is `redshift`) Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. @@ -216,7 +216,7 @@ is redshift). More details are given below. Using `redshift_configuration` requires the user to also specify a `s3_configuration` block. More details are given below. -The `source_configuration` object supports the following: +The `kinesis_source_configuration` object supports the following: * `kinesis_stream_arn` (Required) The kinesis stream used as the source of the firehose delivery stream. * `role_arn` (Required) The ARN of the role that provides access to the source Kinesis stream.