Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support Eventbridge Pipes parameters #31607

Merged
merged 68 commits into from
Jun 26, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
68 commits
Select commit Hold shift + click to select a range
bd781d4
Support Eventbridge Pipes parameters
jordanfinners Apr 16, 2023
8794d92
Merge branch 'main' into HEAD
ewbankkit Jun 9, 2023
e80db56
r/aws_pipes_pipe: Alphabetize attributes.
ewbankkit Jun 9, 2023
76d73ea
Add CHANGELOG entry.
ewbankkit Jun 9, 2023
cc443f8
Tidy up 'enrichment_parameters'.
ewbankkit Jun 9, 2023
bc29dd0
Tidy up 'source_parameters'.
ewbankkit Jun 9, 2023
24b9f39
Tidy up 'target_parameters'.
ewbankkit Jun 9, 2023
3229613
'sage_maker_pipeline' -> 'sagemaker_pipeline'.
ewbankkit Jun 9, 2023
3d5cef3
pipes: Use 'go-cmp' in unit tests.
ewbankkit Jun 9, 2023
270783e
pipes: Use 'reflect.DeepEquals' in unit tests.
ewbankkit Jun 9, 2023
10324c7
Revert "pipes: Use 'reflect.DeepEquals' in unit tests."
ewbankkit Jun 9, 2023
bc9e9a3
Revert "pipes: Use 'go-cmp' in unit tests."
ewbankkit Jun 9, 2023
e93fbba
r/aws_pipes_pipe: 'enrichment_parameters.http_parameters' changes.
ewbankkit Jun 9, 2023
1299833
r/aws_pipes_pipe: 'source_parameters.filter_criteria' changes.
ewbankkit Jun 11, 2023
f67104c
r/aws_pipes_pipe: 'source_parameters.sqs_queue' -> 'source_parameters…
ewbankkit Jun 12, 2023
fc4286f
r/aws_pipes_pipe: 'source_parameters.dynamo_db_stream' -> 'source_par…
ewbankkit Jun 12, 2023
3e6bfec
r/aws_pipes_pipe: 'source_parameters.active_mq_broker' -> 'source_par…
ewbankkit Jun 12, 2023
526fce8
r/aws_pipes_pipe: 'source_parameters.kinesis_stream' -> 'source_param…
ewbankkit Jun 13, 2023
e0b8793
r/aws_pipes_pipe: 'source_parameters.managed_streaming_kafka' -> 'sou…
ewbankkit Jun 13, 2023
f37a79b
r/aws_pipes_pipe: 'source_parameters.rabbit_mq_broker' -> 'source_par…
ewbankkit Jun 13, 2023
0fb4858
r/aws_pipes_pipe: 'source_parameters.self_managed_kafka' -> 'source_p…
ewbankkit Jun 13, 2023
85b0060
r/aws_pipes_pipe: Fill out 'expandUpdatePipeSourceParameters'.
ewbankkit Jun 13, 2023
b2363d3
Update CHANGELOG entry.
ewbankkit Jun 13, 2023
9c7c8c1
r/aws_pipes_pipe: 'target_parameters.step_function' -> 'target_parame…
ewbankkit Jun 13, 2023
ef71c23
r/aws_pipes_pipe: 'target_parameters.sqs_queue' -> 'target_parameters…
ewbankkit Jun 13, 2023
4df4d06
r/aws_pipes_pipe: 'target_parameters.sagemaker_pipeline' -> 'target_p…
ewbankkit Jun 13, 2023
f24f40a
r/aws_pipes_pipe: 'target_parameters.redshift_data' -> 'target_parame…
ewbankkit Jun 13, 2023
c0865b2
r/aws_pipes_pipe: 'target_parameters.lambda_function' -> 'target_para…
ewbankkit Jun 13, 2023
92d6154
r/aws_pipes_pipe: 'target_parameters.kinesis_stream_parameters' -> 't…
ewbankkit Jun 13, 2023
121c107
r/aws_pipes_pipe: Modify 'target_parameters.http_parameters'.
ewbankkit Jun 13, 2023
8080fa9
r/aws_pipes_pipe: 'target_parameters.event_bridge_event_bus' -> 'targ…
ewbankkit Jun 13, 2023
0a20bb7
r/aws_pipes_pipe: 'target_parameters.cloudwatch_logs' -> 'target_para…
ewbankkit Jun 13, 2023
a93645e
r/aws_pipes_pipe: 'target_parameters.batch_target' -> 'target_paramet…
ewbankkit Jun 13, 2023
d77d3c5
r/aws_pipes_pipe: 'target_parameters.ecs_task' -> 'target_parameters.…
ewbankkit Jun 14, 2023
ccd8ab6
r/aws_pipes_pipe: Consolidate source files.
ewbankkit Jun 14, 2023
f1006ce
Merge branch 'main' into HEAD
ewbankkit Jun 14, 2023
24b6f1a
r/aws_pipes_pipe: Remove defaults from 'target_parameters'.
ewbankkit Jun 14, 2023
09188e0
Revert "r/aws_pipes_pipe: Remove defaults from 'target_parameters'."
ewbankkit Jun 14, 2023
0e90c63
r/aws_pipes_pipe: Use 'reflect.ValueOf().IsZero()' to detect empty re…
ewbankkit Jun 14, 2023
f6daaff
r/aws_pipes_pipe: Tidy up acceptance tests.
ewbankkit Jun 14, 2023
23db5f7
r/aws_pipes_pipe: Remove defaults from 'source_parameters'.
ewbankkit Jun 14, 2023
b7c73e9
r/aws_pipes_pipe: Remove defaults from 'target_parameters'.
ewbankkit Jun 14, 2023
7a53bdc
r/aws_pipes_pipe: Some Computed attributes for 'source_parameters'.
ewbankkit Jun 14, 2023
3cdb6b5
r/aws_pipes_pipe: Remove DiffSuppressFunc from 'target_parameters'.
ewbankkit Jun 14, 2023
d84df7c
r/aws_pipes_pipe: Original acceptance tests passing.
ewbankkit Jun 15, 2023
02b9d32
Use 'enum.Slice'.
ewbankkit Jun 19, 2023
2c77f7a
'TestAccPipesPipe_target' -> 'TestAccPipesPipe_targetUpdate'.
ewbankkit Jun 20, 2023
a8ba9f9
r/aws_pipes_pipe: Add 'TestAccPipesPipe_basicKinesis'.
ewbankkit Jun 24, 2023
886249e
Merge branch 'main' into HEAD
ewbankkit Jun 24, 2023
a516120
Add 'types.IsZero'.
ewbankkit Jun 24, 2023
026fde6
r/aws_pipes_pipe: Use 'types.IsZero'.
ewbankkit Jun 24, 2023
c79ef82
r/aws_pipes_pipe: Add 'TestAccPipesPipe_dynamoDBSourceCloudWatchLogsT…
ewbankkit Jun 24, 2023
f00ee8b
r/aws_pipes_pipe: Add 'TestAccPipesPipe_activeMQSourceStepFunctionTar…
ewbankkit Jun 24, 2023
5e2374f
r/aws_pipes_pipe: Add 'TestAccPipesPipe_rabbitMQSourceEventBusTarget'.
ewbankkit Jun 24, 2023
b3bd872
Add 'TestAccPipesPipe_mskSourceHTTPTarget'.
ewbankkit Jun 26, 2023
2181c3e
Add 'TestAccPipesPipe_selfManagedKafkaSourceLambdaFunctionTarget'.
ewbankkit Jun 26, 2023
4dac458
r/aws_pipes_pipe: Add 'TestAccPipesPipe_sqsSourceRedshiftTarget'.
ewbankkit Jun 26, 2023
77bd4d5
Add 'TestAccPipesPipe_sqsSourceSagemakerTarget'.
ewbankkit Jun 26, 2023
cfd5554
% make testacc TESTARGS='-run=TestAccPipesPipe_sqsSourceBatchJobTarge…
ewbankkit Jun 26, 2023
80b2a9b
Add 'TestAccPipesPipe_sqsSourceECSTaskTarget'.
ewbankkit Jun 26, 2023
c3d1cac
r/aws_pipes_pipe: Add 'TestAccPipesPipe_enrichmentParameters'.
ewbankkit Jun 26, 2023
622cb2a
Fix terrafmt errors in acceptance test configurations.
ewbankkit Jun 26, 2023
db237c6
Fix markdown-lint 'MD012/no-multiple-blanks Multiple consecutive blan…
ewbankkit Jun 26, 2023
4d08912
Fix semgrep errors.
ewbankkit Jun 26, 2023
517e8a2
Add NOTE to CHANGELOG entries.
ewbankkit Jun 26, 2023
bbb25ff
Fix golangci-lint 'paralleltest'.
ewbankkit Jun 26, 2023
a2c7ca6
Fix golangci-lint 'unused'.
ewbankkit Jun 26, 2023
969b6a4
Fix golangci-lint 'staticcheck'.
ewbankkit Jun 26, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ require (
github.com/mitchellh/go-testing-interface v1.14.1
github.com/pquerna/otp v1.4.0
github.com/shopspring/decimal v1.3.1
github.com/stretchr/testify v1.8.2
golang.org/x/crypto v0.8.0
golang.org/x/exp v0.0.0-20230206171751-46f607a40771
golang.org/x/tools v0.6.0
Expand Down Expand Up @@ -98,6 +99,7 @@ require (
github.com/bgentry/speakeasy v0.1.0 // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/cloudflare/circl v1.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/evanphx/json-patch v0.5.2 // indirect
github.com/fatih/color v1.14.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
Expand All @@ -122,6 +124,7 @@ require (
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/posener/complete v1.1.1 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/spf13/cast v1.3.1 // indirect
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -329,14 +329,19 @@ github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
Expand Down
1 change: 1 addition & 0 deletions internal/service/pipes/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
* AWS Provider: [Contribution Guide](https://hashicorp.github.io/terraform-provider-aws/#contribute)
* Service User Guide: [Amazon EventBridge Pipes](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes.html)
* Service API Guide: [Welcome](https://docs.aws.amazon.com/eventbridge/latest/pipes-reference/Welcome.html)

185 changes: 185 additions & 0 deletions internal/service/pipes/enrichment_parameters.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
package pipes

import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/pipes/types"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
"github.com/hashicorp/terraform-provider-aws/internal/flex"
)

var enrichment_parameters_schema = &schema.Schema{
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"input_template": {
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringLenBetween(0, 8192),
},
"http_parameters": {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"header": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"key": {
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(0, 512),
},
"value": {
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(0, 512),
},
},
},
},
"path_parameters": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Schema{
Type: schema.TypeString,
},
},
"query_string": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"key": {
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(0, 512),
},
"value": {
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringLenBetween(0, 512),
},
},
},
},
},
},
},
},
},
}

func expandEnrichmentParameters(config []interface{}) *types.PipeEnrichmentParameters {
if len(config) == 0 {
return nil
}

var parameters types.PipeEnrichmentParameters
for _, c := range config {
param := c.(map[string]interface{})
if val, ok := param["input_template"].(string); ok && val != "" {
parameters.InputTemplate = aws.String(val)
}
if val, ok := param["http_parameters"]; ok {
parameters.HttpParameters = expandEnrichmentHTTPParameters(val.([]interface{}))
}
}
return &parameters
}

func expandEnrichmentHTTPParameters(config []interface{}) *types.PipeEnrichmentHttpParameters {
if len(config) == 0 {
return nil
}

var parameters types.PipeEnrichmentHttpParameters
for _, c := range config {
param := c.(map[string]interface{})
if val, ok := param["path_parameters"]; ok {
parameters.PathParameterValues = flex.ExpandStringValueList(val.([]interface{}))
}

if val, ok := param["header"]; ok {
headers := map[string]string{}
if values, ok := val.([]interface{}); ok {
for _, v := range values {
valueParam := v.(map[string]interface{})

if key, ok := valueParam["key"].(string); ok && key != "" {
if value, ok := valueParam["value"].(string); ok && value != "" {
headers[key] = value
}
}
}
}
if len(headers) > 0 {
parameters.HeaderParameters = headers
}
}

if val, ok := param["query_string"]; ok {
queryStrings := map[string]string{}
if values, ok := val.([]interface{}); ok {
for _, v := range values {
valueParam := v.(map[string]interface{})

if key, ok := valueParam["key"].(string); ok && key != "" {
if value, ok := valueParam["value"].(string); ok && value != "" {
queryStrings[key] = value
}
}
}
}
if len(queryStrings) > 0 {
parameters.QueryStringParameters = queryStrings
}
}
}
return &parameters
}

func flattenEnrichmentParameters(enrichmentParameters *types.PipeEnrichmentParameters) []map[string]interface{} {
config := make(map[string]interface{})

if enrichmentParameters.InputTemplate != nil {
config["input_template"] = *enrichmentParameters.InputTemplate
}

if enrichmentParameters.HttpParameters != nil {
httpParameters := make(map[string]interface{})

var headerParameters []map[string]interface{}
for key, value := range enrichmentParameters.HttpParameters.HeaderParameters {
header := make(map[string]interface{})
header["key"] = key
header["value"] = value
headerParameters = append(headerParameters, header)
}
httpParameters["header"] = headerParameters

var queryStringParameters []map[string]interface{}
for key, value := range enrichmentParameters.HttpParameters.QueryStringParameters {
queryString := make(map[string]interface{})
queryString["key"] = key
queryString["value"] = value
queryStringParameters = append(queryStringParameters, queryString)
}
httpParameters["query_string"] = queryStringParameters
httpParameters["path_parameters"] = flex.FlattenStringValueList(enrichmentParameters.HttpParameters.PathParameterValues)

config["http_parameters"] = []map[string]interface{}{httpParameters}
}

if len(config) == 0 {
return nil
}

result := []map[string]interface{}{config}
return result
}
143 changes: 143 additions & 0 deletions internal/service/pipes/enrichment_parameters_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
package pipes

import (
"testing"

"github.com/aws/aws-sdk-go-v2/service/pipes/types"
"github.com/aws/aws-sdk-go/aws"
"github.com/stretchr/testify/assert"
)

func Test_expandEnrichmentParameters(t *testing.T) {
tests := map[string]struct {
config map[string]interface{}
expected *types.PipeEnrichmentParameters
}{
"input_template config": {
config: map[string]interface{}{
"input_template": "some template",
},
expected: &types.PipeEnrichmentParameters{
InputTemplate: aws.String("some template"),
},
},
"http_parameters config": {
config: map[string]interface{}{
"http_parameters": []interface{}{
map[string]interface{}{
"path_parameters": []interface{}{"a", "b"},
"header": []interface{}{
map[string]interface{}{
"key": "key1",
"value": "value1",
},
map[string]interface{}{
"key": "key2",
"value": "value2",
},
},
"query_string": []interface{}{
map[string]interface{}{
"key": "key3",
"value": "value3",
},
map[string]interface{}{
"key": "key4",
"value": "value4",
},
},
},
},
},
expected: &types.PipeEnrichmentParameters{
HttpParameters: &types.PipeEnrichmentHttpParameters{
PathParameterValues: []string{"a", "b"},
HeaderParameters: map[string]string{
"key1": "value1",
"key2": "value2",
},
QueryStringParameters: map[string]string{
"key3": "value3",
"key4": "value4",
},
},
},
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
got := expandEnrichmentParameters([]interface{}{tt.config})

assert.Equal(t, tt.expected, got)
})
}
}

func Test_flattenEnrichmentParameters(t *testing.T) {
tests := map[string]struct {
config *types.PipeEnrichmentParameters
expected []map[string]interface{}
}{
"input_template config": {
config: &types.PipeEnrichmentParameters{
InputTemplate: aws.String("some template"),
},
expected: []map[string]interface{}{
{
"input_template": "some template",
},
},
},
"http_parameters config": {
config: &types.PipeEnrichmentParameters{
HttpParameters: &types.PipeEnrichmentHttpParameters{
PathParameterValues: []string{"a", "b"},
HeaderParameters: map[string]string{
"key1": "value1",
"key2": "value2",
},
QueryStringParameters: map[string]string{
"key3": "value3",
"key4": "value4",
},
},
},
expected: []map[string]interface{}{
{
"http_parameters": []map[string]interface{}{
{
"path_parameters": []interface{}{"a", "b"},
"header": []map[string]interface{}{
{
"key": "key1",
"value": "value1",
},
{
"key": "key2",
"value": "value2",
},
},
"query_string": []map[string]interface{}{
{
"key": "key3",
"value": "value3",
},
{
"key": "key4",
"value": "value4",
},
},
},
},
},
},
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
got := flattenEnrichmentParameters(tt.config)

assert.Equal(t, tt.expected, got)
})
}
}
Loading