minimal kinesis delivery stream configuration

JamesWoolfenden · Jul 17, 2024 · c087864 · c087864
1 parent 09cc252
commit c087864
Show file tree

Hide file tree

Showing 7 changed files with 135 additions and 3 deletions.
diff --git a/.run/test sato parse.run.xml b/.run/test sato parse.run.xml
@@ -2,7 +2,7 @@
   <configuration default="false" name="test sato parse" type="GoApplicationRunConfiguration" factoryName="Go Application">
     <module name="sato" />
     <working_directory value="$PROJECT_DIR$" />
-    <parameters value="parse -f $USER_HOME$/test/cf/athena.yaml" />
+    <parameters value="parse -f $USER_HOME$/test/cf/kinesis.yaml" />
     <kind value="PACKAGE" />
     <package value="sato" />
     <directory value="$PROJECT_DIR$" />

diff --git a/examples/kinesis.yaml b/examples/kinesis.yaml
@@ -0,0 +1,32 @@
+Resources:
+  SecHubEventsFirehoseDeliveryStream:
+    Type: AWS::KinesisFirehose::DeliveryStream
+    Properties:
+      DeliveryStreamName: aws-sechub-logs-kinesis-delivery-stream
+      DeliveryStreamType: DirectPut
+      DeliveryStreamEncryptionConfigurationInput:
+        KeyType: AWS_OWNED_CMK
+      ExtendedS3DestinationConfiguration:
+        BucketARN: !GetAtt S3Bucketsechubyashdatafirehose.Arn
+        Prefix: !Sub "sechub_events/managementid=${AWS::AccountId}/!{timestamp:yyyy}/!{timestamp:MM}/!{timestamp:dd}/"
+        ErrorOutputPrefix: "aws-sechub-logs-firehose-error/"
+        RoleARN: !GetAtt kinesisFirehoseRole.Arn
+        CloudWatchLoggingOptions:
+          Enabled: true
+          LogGroupName: !Ref FirehoseLogGroup
+          LogStreamName: !Ref FirehoseLogStream
+        BufferingHints:
+          IntervalInSeconds: 900
+          SizeInMBs: 30
+        CompressionFormat: "GZIP"
+        ProcessingConfiguration:
+          Enabled: true
+          Processors:
+            - Type: Lambda
+              Parameters:
+                - ParameterName: LambdaArn
+                  ParameterValue: !GetAtt LambdaFunctiontransformationsechub.Arn
+                - ParameterName: BufferIntervalInSeconds
+                  ParameterValue: "600"
+                - ParameterName: BufferSizeInMBs
+                  ParameterValue: "3"
diff --git a/main.go b/main.go
@@ -40,7 +40,7 @@ func main() {
 				Action: func(*cli.Context) error {
 					err := cf.Parse(file, destination)
 					if err != nil {
-						fmt.Println("parse failure", err)
+						log.Error().Err(err)
 					}
 
 					return nil

diff --git a/src/cf/lookup.go b/src/cf/lookup.go
@@ -121,6 +121,7 @@ func lookup(myType string) []byte {
 		"AWS::IAM::Role":                                   awsIamRole,
 		"AWS::IAM::User":                                   awsIamUser,
 		"AWS::IAM::UserToGroupAddition":                    awsIamGroupMembership,
+		"AWS::KinesisFirehose::DeliveryStream":             awsKinesisFirehoseDeliveryStream,
 		"AWS::KMS::Alias":                                  awskmsAlias,
 		"AWS::KMS::Key":                                    awsKmsKey,
 		"AWS::Lambda::EventSourceMapping":                  awsLambdaEventSourceMapping,

diff --git a/src/cf/parse.go b/src/cf/parse.go
@@ -45,7 +45,7 @@ func Parse(file string, destination string) error {
 	// Open a cloudFormation from file (can be JSON or YAML)
 	fileAbs, err := filepath.Abs(file)
 	if err != nil {
-		return fmt.Errorf("filpath failure %w", err)
+		return fmt.Errorf("filepath failure %w", err)
 	}
 
 	cloudFormation, err := goformation.Open(fileAbs)

diff --git a/src/cf/resources.go b/src/cf/resources.go
@@ -351,3 +351,6 @@ var awsAthenaWorkGroup []byte
 
 //go:embed resources/aws_athena_named_query.template
 var awsAthenaNamedQuery []byte
+
+//go:embed resources/aws_kinesis_firehose_delivery_stream.template
+var awsKinesisFirehoseDeliveryStream []byte
diff --git a/src/cf/resources/aws_kinesis_firehose_delivery_stream.template b/src/cf/resources/aws_kinesis_firehose_delivery_stream.template
@@ -0,0 +1,96 @@
+resource "aws_kinesis_firehose_delivery_stream" "{{.item}}" {
+    name        = "{{.resource.DeliveryStreamName}}"
+    destination = "extended_s3"
+
+{{- if .resource.DeliveryStreamEncryptionConfigurationInput}}
+    server_side_encryption {
+        enabled = true
+        {{- if .resource.DeliveryStreamEncryptionConfigurationInput.KeyARN}}
+        key_arn =  {{.resource.DeliveryStreamEncryptionConfigurationInput.KeyARN|Quote}}
+        {{- end}}
+        {{- if .resource.DeliveryStreamEncryptionConfigurationInput.KeyType}}
+        key_type = {{.resource.DeliveryStreamEncryptionConfigurationInput.KeyType|Quote}}
+        {{- end}}
+    }
+{{- end }}
+
+{{- if .resource.ExtendedS3DestinationConfiguration  }}
+    extended_s3_configuration {
+        role_arn   = aws_iam_role.firehose_role.arn
+        bucket_arn = {{ .resource.ExtendedS3DestinationConfiguration.BucketARN|Quote }}
+{{- if .resource.ExtendedS3DestinationConfiguration.BufferingHints.SizeInMBs}}
+        buffering_size= {{ .resource.ExtendedS3DestinationConfiguration.BufferingHints.SizeInMBs}}
+{{- end }}
+{{- if .resource.ExtendedS3DestinationConfiguration.BufferingHints.IntervalInSeconds}}
+        buffering_interval= {{.resource.ExtendedS3DestinationConfiguration.BufferingHints.IntervalInSeconds}}
+{{- end}}
+
+{{- if .resource.ExtendedS3DestinationConfiguration.CloudWatchLoggingOptions}}
+        cloudwatch_logging_options{
+            enabled = {{Boolean .resource.ExtendedS3DestinationConfiguration.CloudWatchLoggingOptions.Enabled}}
+{{- if .resource.ExtendedS3DestinationConfiguration.CloudWatchLoggingOptions.LogGroupName}}
+            log_group_name = {{.resource.ExtendedS3DestinationConfiguration.CloudWatchLoggingOptions.LogGroupName|Quote}}
+{{- end}}
+{{- if .resource.ExtendedS3DestinationConfiguration.CloudWatchLoggingOptions.LogStreamName}}
+            log_stream_name = {{.resource.ExtendedS3DestinationConfiguration.CloudWatchLoggingOptions.LogStreamName|Quote}}
+{{- end}}
+        }
+{{- end}}
+{{- if .resource.ExtendedS3DestinationConfiguration.CompressionFormat}}
+        compression_format = {{.resource.ExtendedS3DestinationConfiguration.CompressionFormat}}
+{{- end}}
+{{- if .resource.ExtendedS3DestinationConfiguration.CustomTimeZone}}
+        custom_timezone = {{.resource.ExtendedS3DestinationConfiguration.CustomTimeZone}}
+{{- end }}
+{{- if .resource.ExtendedS3DestinationConfiguration.DataFormatConversionConfiguration}}
+        data_format_conversion_configuration {
+        }
+{{- end}}
+{{- if .resource.ExtendedS3DestinationConfiguration.DynamicPartitioningConfiguration}}
+        dynamic_partitioning_configuration {
+        }
+{{- end}}
+    {{- if .resource.ExtendedS3DestinationConfiguration.ErrorOutputPrefix}}
+        error_output_prefix = {{.resource.ExtendedS3DestinationConfiguration.ErrorOutputPrefix|Quote}}
+    {{- end }}
+{{- if .resource.ExtendedS3DestinationConfiguration.RoleARN}}
+    role_arn = {{.resource.ExtendedS3DestinationConfiguration.RoleARN|Quote}}
+{{- end}}
+{{- if  .resource.ExtendedS3DestinationConfiguration.ProcessingConfiguration }}
+        processing_configuration {
+            enabled = {{ Boolean .resource.ExtendedS3DestinationConfiguration.ProcessingConfiguration.Enabled}}
+
+{{- range $a, $i := .resource.ExtendedS3DestinationConfiguration.ProcessingConfiguration.Processors}}
+            processors {
+                type = {{$i.Type}}
+
+{{- range $b, $j := $i.Parameters}}
+                parameters {
+                    parameter_name  = {{ $j.ParameterName|Quote}}
+                    parameter_value = {{ $j.ParameterValue|Quote}}
+                    }
+{{- end}}
+                }
+{{- end}}
+            }
+{{- if .resource.ExtendedS3DestinationConfiguration.S3BackupMode}}
+            s3_backup_mode = {{.resource.ExtendedS3DestinationConfiguration.S3BackupMode}}
+{{- end}}
+{{- if .resource.ExtendedS3DestinationConfiguration.S3BackupConfiguration}}
+            s3_backup_configuration {
+                bucket_arn = {{.resource.ExtendedS3DestinationConfiguration.S3BackupConfiguration.BucketARN|Quote}}
+                role_arn   = {{.resource.ExtendedS3DestinationConfiguration.S3BackupConfiguration.RoleARN|Quote}}
+                prefix     = {{.resource.ExtendedS3DestinationConfiguration.S3BackupConfiguration.Prefix|Quote}}
+            }
+{{- end}}
+
+{{- end}}
+    }
+{{- end}}
+
+  {{- if .resource.Tags}}
+    tags = {
+       {{Tags .resource.Tags}}
+    }
+  {{- end}}
+}