[Bug]: HTTP 500 Internal error logs/ListTagsForResource for Localstack

[ifzal-ahmed-npl]

Terraform Core Version

1.3.8

AWS Provider Version

4.48.0

Affected Resource(s)

aws_cloudwatch_log_group

Expected Behavior

aws_cloudwatch_log_group resource to be created

Actual Behavior

Terraform hangs when trying to create the resource in an infinite loop

Relevant Error/Panic Output Snippet

No response

Terraform Configuration Files

data "aws_caller_identity" "current" {}
#Create a role
resource "aws_iam_role" "ec2_role" {
name = "${var.env}-ec2-role"

assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
EOF

tags = {
Platform = "Test"
Service = "Test"
"Application Service" = "TEST"
Legacy = "No"
}
}

#Create a policy
resource "aws_iam_role_policy" "ec2_policy" {
name = "${var.env}-ec2-policy"
role = aws_iam_role.ec2_role.id

policy = <<EOF
{
"Statement": [
{
"Action": [
"s3:GetObject",
"s3:List*"
],
"Effect": "Allow",
"Resource": [
"*"
]
}
],
"Version": "2012-10-17"
}
EOF
}

resource "aws_iam_role" "ecs_execution_role" {
name = "${var.env}-task-execution-role"

assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ecs-tasks.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
EOF

tags = {
Platform = "Test"
Service = "Test"
"Application Service" = "TEST"
Legacy = "No"
}
}

#Create a policy and attach to role
resource "aws_iam_role_policy" "ecs_execution_role_policy" {
name = "${var.env}-task-execution-policy"
role = aws_iam_role.ecs_execution_role.id

policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"logs:CreateLogStream",
"logs:PutLogEvents",
"s3:",
"s3-object-lambda:"
],
"Resource": ""
},
{
"Effect": "Allow",
"Action": [
"kms:GenerateDataKey",
"kms:Decrypt",
"kms:Encrypt",
"ssm:GetParameter",
"ssm:GetParameters",
"sqs:DeleteMessage",
"sqs:GetQueueUrl",
"sqs:ReceiveMessage",
"sqs:SendMessage"
],
"Resource": [
"${aws_kms_key.kms_key.arn}"
]
},
{
"Sid": "StateMachineResponses",
"Effect": "Allow",
"Action": [
"states:SendTaskSuccess",
"states:SendTaskFailure",
"states:SendTaskHeartbeat"
],
"Resource": ""
}
]
}
EOF
}

resource "aws_kms_key" "kms_key" {

description = "KMS key"
enable_key_rotation = true
policy = <<EOF
{
"Version": "2012-10-17",
"Id": "key-consolepolicy-2",
"Statement": [
{
"Sid": "Enable IAM User Permissions",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"
},
"Action": "kms:",
"Resource": ""
},
{
"Sid": "Allow access for Key Administrators",
"Effect": "Allow",
"Principal": {
"AWS": [
"arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"
]
},
"Action": [
"kms:Create*",
"kms:Describe*",
"kms:Enable*",
"kms:List*",
"kms:Put*",
"kms:Update*",
"kms:Revoke*",
"kms:Disable*",
"kms:Get*",
"kms:Delete*",
"kms:TagResource",
"kms:UntagResource",
"kms:ScheduleKeyDeletion",
"kms:CancelKeyDeletion"
],
"Resource": ""
},
{
"Sid": "Allow use of the key",
"Effect": "Allow",
"Principal": {
"Service": "logs.eu-west-1.amazonaws.com",
"AWS": [
"arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS",
"${aws_iam_role.ec2_role.arn}",
"${aws_iam_role.ecs_execution_role.arn}"
]
},
"Action": [
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt",
"kms:GenerateDataKey*",
"kms:DescribeKey"
],
"Resource": ""
},
{
"Sid": "Allow attachment of persistent resources",
"Effect": "Allow",
"Principal": {
"AWS": [
"arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS",
"${aws_iam_role.ec2_role.arn}",
"${aws_iam_role.ecs_execution_role.arn}"
]
},
"Action": [
"kms:CreateGrant",
"kms:ListGrants",
"kms:RevokeGrant"
],
"Resource": "",
"Condition": {
"Bool": {
"kms:GrantIsForAWSResource": "true"
}
}
}
]
}
EOF

tags = {
Platform = "test"
Service = "test"
"Application Service" = "TEST"
Legacy = "No"
Process = "Test"
}

}

resource "aws_cloudwatch_log_group" "test_logs" {
name = "test-logs-${var.env}"
retention_in_days = 14
kms_key_id = aws_kms_key.kms_key.arn
}

Steps to Reproduce

• Make sure you have Localstack running and set TF_LOG to DEBUG
• Run a terraform init with aws provider version 4.48.0
• Run a terraform plan & apply
• The terraform should hang on creation along with 500 errors appearing in the output

Debug Output

2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: [DEBUG] [aws-sdk-go] DEBUG: Request logs/ListTagsForResource Details:
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: ---[ REQUEST POST-SIGN ]-----------------------------
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: POST / HTTP/1.1
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Host: localhost:4566
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: User-Agent: APN/1.0 HashiCorp/1.0 Terraform/1.3.6 (+https://www.terraform.io) terraform-provider-aws/4.48.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go/1.44.162 (go1.19.3; linux; amd64)
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Content-Length: 112
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Authorization: AWS4-HMAC-SHA256 Credential=test/20221220/us-east-1/logs/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-target, Signature=263d5eccbb57d7e0756aee3ca4124e17355a54cfd49bdf936f6b0a12157a2476
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Content-Type: application/x-amz-json-1.1
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: X-Amz-Date: 20221220T155334Z
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: X-Amz-Target: Logs_20140328.ListTagsForResource
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Accept-Encoding: gzip
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5:
2022-12-20T15:53:34.928Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: {"resourceArn":"arn:aws:logs:us-east-1:000000000000:log-group:/aws/vendedlogs/bdm-dev-040-curated-to-analytics"}
2022-12-20T15:53:34.929Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: -----------------------------------------------------
2022-12-20T15:53:34.940Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: [DEBUG] [aws-sdk-go] DEBUG: Response logs/ListTagsForResource Details:
2022-12-20T15:53:34.940Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: ---[ RESPONSE ]--------------------------------------
2022-12-20T15:53:34.940Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: HTTP/1.1 500
2022-12-20T15:53:34.940Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Connection: close
2022-12-20T15:53:34.940Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Content-Length: 154
2022-12-20T15:53:34.940Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Access-Control-Allow-Headers: authorization,cache-control,content-length,content-md5,content-type,etag,location,x-amz-acl,x-amz-content-sha256,x-amz-date,x-amz-request-id,x-amz-security-token,x-amz-tagging,x-amz-target,x-amz-user-agent,x-amz-version-id,x-amzn-requestid,x-localstack-target,amz-sdk-invocation-id,amz-sdk-request
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Access-Control-Allow-Methods: HEAD,GET,PUT,POST,DELETE,OPTIONS,PATCH
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Access-Control-Allow-Origin: *
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Access-Control-Expose-Headers: etag,x-amz-version-id
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Content-Type: application/json
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Date: Tue, 20 Dec 2022 15:53:34 GMT
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: Server: hypercorn-h11
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: X-Amzn-Errortype: InternalError
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: X-Amzn-Requestid: VWRAGLARJENDUTX8UO1F0P53VKCY7RPOUTYPADOXXH7JF5MZ8WG8
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5:
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5:
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: -----------------------------------------------------
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: [DEBUG] [aws-sdk-go] {"__type": "InternalError", "message": "exception while calling logs with unknown operation: An unknown error occurred when trying to parse the request."}
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: [DEBUG] [aws-sdk-go] DEBUG: Validate Response logs/ListTagsForResource failed, attempt 11/25, error InternalError: exception while calling logs with unknown operation: An unknown error occurred when trying to parse the request.
2022-12-20T15:53:34.941Z [DEBUG] provider.terraform-provider-aws_v4.48.0_x5: status code: 500, request id: VWRAGLARJENDUTX8UO1F0P53VKCY7RPOUTYPADOXXH7JF5MZ8WG8

Panic Output

No response

Important Factoids

No response

References

No response

Would you like to implement a fix?

None

[github-actions]

Community Note

Voting for Prioritization

• Please vote on this issue by adding a 👍 reaction to the original post to help the community and maintainers prioritize this request.
• Please see our prioritization guide for information on how we prioritize.
• Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request.

Volunteering to Work on This Issue

• If you are interested in working on this issue, please leave a comment.
• If this would be your first contribution, please review the contribution guide.

[ifzal-ahmed-npl]

This error didn't appear in provider version 4.46.0. This bug also affects provider version 4.47.0

[ewbankkit]

Relates #28359.

[github-actions]

I'm going to lock this issue because it has been closed for 30 days ⏳. This helps our maintainers find and focus on the active issues.
If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further.