From ba250f3a042d60a19d925de6b45bda6ff505e0a9 Mon Sep 17 00:00:00 2001 From: Florian Stadler Date: Thu, 23 May 2024 11:12:45 +0200 Subject: [PATCH 1/8] Remove lambda serialization --- ...-Lambda-Function-resource-operations.patch | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 patches/0059-Parallelize-Lambda-Function-resource-operations.patch diff --git a/patches/0059-Parallelize-Lambda-Function-resource-operations.patch b/patches/0059-Parallelize-Lambda-Function-resource-operations.patch new file mode 100644 index 00000000000..165dd855708 --- /dev/null +++ b/patches/0059-Parallelize-Lambda-Function-resource-operations.patch @@ -0,0 +1,47 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Florian Stadler +Date: Wed, 22 May 2024 17:01:32 +0200 +Subject: [PATCH] Parallelize Lambda Function resource operations + +Upstream introduced serialization of Lambda Function resource +operations to fight high memory usage when managing a lot of +Lambda functions. +We think this was an optimization for a special edge case that +drastically worsens the UX for the majority of users. + +diff --git a/internal/service/lambda/function.go b/internal/service/lambda/function.go +index 4c49c5fb21..f2ce387f67 100644 +--- a/internal/service/lambda/function.go ++++ b/internal/service/lambda/function.go +@@ -35,7 +35,6 @@ import ( + + const ( + FunctionVersionLatest = "$LATEST" +- mutexKey = `aws_lambda_function` + listVersionsMaxItems = 10000 + ) + +@@ -481,11 +480,6 @@ func resourceFunctionCreate(ctx context.Context, d *schema.ResourceData, meta in + } + + if v, ok := d.GetOk("filename"); ok { +- // Grab an exclusive lock so that we're only reading one function into memory at a time. +- // See https://github.com/hashicorp/terraform/issues/9364. +- conns.GlobalMutexKV.Lock(mutexKey) +- defer conns.GlobalMutexKV.Unlock(mutexKey) +- + zipFile, err := readFileContents(v.(string)) + + if err != nil { +@@ -942,11 +936,6 @@ func resourceFunctionUpdate(ctx context.Context, d *schema.ResourceData, meta in + } + + if v, ok := d.GetOk("filename"); ok { +- // Grab an exclusive lock so that we're only reading one function into memory at a time. +- // See https://github.com/hashicorp/terraform/issues/9364 +- conns.GlobalMutexKV.Lock(mutexKey) +- defer conns.GlobalMutexKV.Unlock(mutexKey) +- + zipFile, err := readFileContents(v.(string)) + + if err != nil { From d8152272c49132cd7839aa16b179eea3178c8500 Mon Sep 17 00:00:00 2001 From: Florian Stadler Date: Thu, 23 May 2024 11:26:44 +0200 Subject: [PATCH 2/8] Add tests for replicating lambda serialization problems --- examples/parallel-lambdas-s3/Pulumi.yaml | 3 + examples/parallel-lambdas-s3/create_lambda.sh | 23 +++++++ examples/parallel-lambdas-s3/index.ts | 68 +++++++++++++++++++ examples/parallel-lambdas-s3/package.json | 16 +++++ examples/parallel-lambdas-s3/tsconfig.json | 18 +++++ examples/parallel-lambdas/Pulumi.yaml | 3 + examples/parallel-lambdas/create_lambda.sh | 23 +++++++ examples/parallel-lambdas/index.ts | 45 ++++++++++++ examples/parallel-lambdas/package.json | 16 +++++ examples/parallel-lambdas/tsconfig.json | 18 +++++ 10 files changed, 233 insertions(+) create mode 100644 examples/parallel-lambdas-s3/Pulumi.yaml create mode 100755 examples/parallel-lambdas-s3/create_lambda.sh create mode 100644 examples/parallel-lambdas-s3/index.ts create mode 100644 examples/parallel-lambdas-s3/package.json create mode 100644 examples/parallel-lambdas-s3/tsconfig.json create mode 100644 examples/parallel-lambdas/Pulumi.yaml create mode 100755 examples/parallel-lambdas/create_lambda.sh create mode 100644 examples/parallel-lambdas/index.ts create mode 100644 examples/parallel-lambdas/package.json create mode 100644 examples/parallel-lambdas/tsconfig.json diff --git a/examples/parallel-lambdas-s3/Pulumi.yaml b/examples/parallel-lambdas-s3/Pulumi.yaml new file mode 100644 index 00000000000..372cae9dd9b --- /dev/null +++ b/examples/parallel-lambdas-s3/Pulumi.yaml @@ -0,0 +1,3 @@ +name: kms-alias +runtime: nodejs +description: Parallel Lambdas example diff --git a/examples/parallel-lambdas-s3/create_lambda.sh b/examples/parallel-lambdas-s3/create_lambda.sh new file mode 100755 index 00000000000..b40e79ea62c --- /dev/null +++ b/examples/parallel-lambdas-s3/create_lambda.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euo pipefail + +temp_dir=$(mktemp -d) +trap "rm -rf $temp_dir" EXIT +pushd $temp_dir + +printf "const { version } = require(\"@aws-sdk/client-s3/package.json\");\n\nexports.handler = async () => ({ version });\n" > index.js +dd if=/dev/urandom of=example.txt bs=25m count=1 +zip lambda.zip index.js example.txt + +# Check the size of lambda.zip +zip_size=$(stat -f %z lambda.zip) + +# Compare the size with 25MB (25 * 1024 * 1024 bytes) +if [[ $zip_size -lt $((25 * 1024 * 1024)) ]]; then + echo "lambda.zip is smaller than 25MB. Actual size $zip_size." + exit 1 +fi + +popd +mv $temp_dir/lambda.zip . diff --git a/examples/parallel-lambdas-s3/index.ts b/examples/parallel-lambdas-s3/index.ts new file mode 100644 index 00000000000..cb29625ed17 --- /dev/null +++ b/examples/parallel-lambdas-s3/index.ts @@ -0,0 +1,68 @@ +// Copyright 2016-2018, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as pulumi from "@pulumi/pulumi"; +import * as aws from "@pulumi/aws"; +import * as std from "@pulumi/std"; + +const config = new pulumi.Config("aws"); +const providerOpts = { provider: new aws.Provider("prov", { region: config.require("envRegion") }) }; + +const assumeRole = aws.iam.getPolicyDocument({ + statements: [{ + effect: "Allow", + principals: [{ + type: "Service", + identifiers: ["lambda.amazonaws.com"], + }], + actions: ["sts:AssumeRole"], + }], +}, providerOpts); +const role = new aws.iam.Role("parallel-iam-role", { + assumeRolePolicy: assumeRole.then(assumeRole => assumeRole.json) +}, providerOpts); + +const bucket = new aws.s3.BucketV2("lambda-code-storage", { + forceDestroy: true, +}, providerOpts); + +const versioning = new aws.s3.BucketVersioningV2("lambda-code-storage", { + bucket: bucket.id, + versioningConfiguration: { + status: "Enabled", + }, +}, providerOpts); + +for (let i = 0; i < 50; i++) { + const lambdaCode = new aws.s3.BucketObjectv2(`lambda-zip-${i}`, { + bucket: bucket.bucket, + key: `lambda-${i}.zip`, + source: new pulumi.asset.FileAsset("lambda.zip"), + sourceHash: std.filesha256({ + input: "lambda.zip", + }).then(invoke => invoke.result), + }, {...providerOpts, dependsOn: [versioning]}); + + new aws.lambda.Function(`lambda-${i}`, { + role: role.arn, + handler: "index.handler", + runtime: "nodejs20.x", + s3Bucket: bucket.bucket, + s3Key: lambdaCode.key, + s3ObjectVersion: lambdaCode.versionId, + }, { + ...providerOpts, + ignoreChanges: ["replacementSecurityGroupIds"], + }); +} diff --git a/examples/parallel-lambdas-s3/package.json b/examples/parallel-lambdas-s3/package.json new file mode 100644 index 00000000000..b88deddba3d --- /dev/null +++ b/examples/parallel-lambdas-s3/package.json @@ -0,0 +1,16 @@ +{ + "name": "parallel-lambdas-s3", + "version": "0.0.1", + "license": "Apache-2.0", + "scripts": { + "build": "tsc" + }, + "dependencies": { + "@pulumi/pulumi": "^3.0.0", + "@pulumi/aws": "^6.0.0", + "@pulumi/std": "^1.6.2" + }, + "devDependencies": { + "@types/node": "^8.0.0" + } +} diff --git a/examples/parallel-lambdas-s3/tsconfig.json b/examples/parallel-lambdas-s3/tsconfig.json new file mode 100644 index 00000000000..ab65afa6135 --- /dev/null +++ b/examples/parallel-lambdas-s3/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "strict": true, + "outDir": "bin", + "target": "es2016", + "module": "commonjs", + "moduleResolution": "node", + "sourceMap": true, + "experimentalDecorators": true, + "pretty": true, + "noFallthroughCasesInSwitch": true, + "noImplicitReturns": true, + "forceConsistentCasingInFileNames": true + }, + "files": [ + "index.ts" + ] +} diff --git a/examples/parallel-lambdas/Pulumi.yaml b/examples/parallel-lambdas/Pulumi.yaml new file mode 100644 index 00000000000..372cae9dd9b --- /dev/null +++ b/examples/parallel-lambdas/Pulumi.yaml @@ -0,0 +1,3 @@ +name: kms-alias +runtime: nodejs +description: Parallel Lambdas example diff --git a/examples/parallel-lambdas/create_lambda.sh b/examples/parallel-lambdas/create_lambda.sh new file mode 100755 index 00000000000..b40e79ea62c --- /dev/null +++ b/examples/parallel-lambdas/create_lambda.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euo pipefail + +temp_dir=$(mktemp -d) +trap "rm -rf $temp_dir" EXIT +pushd $temp_dir + +printf "const { version } = require(\"@aws-sdk/client-s3/package.json\");\n\nexports.handler = async () => ({ version });\n" > index.js +dd if=/dev/urandom of=example.txt bs=25m count=1 +zip lambda.zip index.js example.txt + +# Check the size of lambda.zip +zip_size=$(stat -f %z lambda.zip) + +# Compare the size with 25MB (25 * 1024 * 1024 bytes) +if [[ $zip_size -lt $((25 * 1024 * 1024)) ]]; then + echo "lambda.zip is smaller than 25MB. Actual size $zip_size." + exit 1 +fi + +popd +mv $temp_dir/lambda.zip . diff --git a/examples/parallel-lambdas/index.ts b/examples/parallel-lambdas/index.ts new file mode 100644 index 00000000000..e302fec1db4 --- /dev/null +++ b/examples/parallel-lambdas/index.ts @@ -0,0 +1,45 @@ +// Copyright 2016-2018, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as pulumi from "@pulumi/pulumi"; +import * as aws from "@pulumi/aws"; + +const config = new pulumi.Config("aws"); +const providerOpts = { provider: new aws.Provider("prov", { region: config.require("envRegion") }) }; + +const assumeRole = aws.iam.getPolicyDocument({ + statements: [{ + effect: "Allow", + principals: [{ + type: "Service", + identifiers: ["lambda.amazonaws.com"], + }], + actions: ["sts:AssumeRole"], + }], +}, providerOpts); +const role = new aws.iam.Role("parallel-iam-role", { + assumeRolePolicy: assumeRole.then(assumeRole => assumeRole.json) +}, providerOpts); + +for (let i = 0; i < 50; i++) { + new aws.lambda.Function(`lambda-${i}`, { + code: new pulumi.asset.FileArchive("lambda.zip"), + role: role.arn, + handler: "index.handler", + runtime: "nodejs20.x", + }, { + ...providerOpts, + ignoreChanges: ["replacementSecurityGroupIds"], + }); +} diff --git a/examples/parallel-lambdas/package.json b/examples/parallel-lambdas/package.json new file mode 100644 index 00000000000..4ac700eeec2 --- /dev/null +++ b/examples/parallel-lambdas/package.json @@ -0,0 +1,16 @@ +{ + "name": "parallel-lambdas", + "version": "0.0.1", + "license": "Apache-2.0", + "scripts": { + "build": "tsc" + }, + "dependencies": { + "@pulumi/pulumi": "^3.0.0", + "@pulumi/aws": "^6.0.0", + "@pulumi/std": "^1.6.2" + }, + "devDependencies": { + "@types/node": "^8.0.0" + } +} diff --git a/examples/parallel-lambdas/tsconfig.json b/examples/parallel-lambdas/tsconfig.json new file mode 100644 index 00000000000..ab65afa6135 --- /dev/null +++ b/examples/parallel-lambdas/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "strict": true, + "outDir": "bin", + "target": "es2016", + "module": "commonjs", + "moduleResolution": "node", + "sourceMap": true, + "experimentalDecorators": true, + "pretty": true, + "noFallthroughCasesInSwitch": true, + "noImplicitReturns": true, + "forceConsistentCasingInFileNames": true + }, + "files": [ + "index.ts" + ] +} From 0c4044c811d667c3283f4d60afefaa3e054c8471 Mon Sep 17 00:00:00 2001 From: Florian Stadler Date: Fri, 24 May 2024 09:59:37 +0200 Subject: [PATCH 3/8] Remove HTTP logger from Lambda resource --- ...ove-HTTP-logger-from-Lambda-resource.patch | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 patches/0060-Remove-HTTP-logger-from-Lambda-resource.patch diff --git a/patches/0060-Remove-HTTP-logger-from-Lambda-resource.patch b/patches/0060-Remove-HTTP-logger-from-Lambda-resource.patch new file mode 100644 index 00000000000..768431dc82f --- /dev/null +++ b/patches/0060-Remove-HTTP-logger-from-Lambda-resource.patch @@ -0,0 +1,29 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Florian Stadler +Date: Fri, 24 May 2024 09:44:07 +0200 +Subject: [PATCH] Remove HTTP logger from Lambda resource + + +diff --git a/internal/service/lambda/service_package_extra.go b/internal/service/lambda/service_package_extra.go +index 54f6aac15a..daa656b829 100644 +--- a/internal/service/lambda/service_package_extra.go ++++ b/internal/service/lambda/service_package_extra.go +@@ -6,6 +6,7 @@ import ( + aws_sdkv2 "github.com/aws/aws-sdk-go-v2/aws" + retry_sdkv2 "github.com/aws/aws-sdk-go-v2/aws/retry" + lambda_sdkv2 "github.com/aws/aws-sdk-go-v2/service/lambda" ++ "github.com/aws/smithy-go/middleware" + tfawserr_sdkv2 "github.com/hashicorp/aws-sdk-go-base/v2/tfawserr" + "github.com/hashicorp/terraform-provider-aws/internal/conns" + "github.com/hashicorp/terraform-provider-aws/names" +@@ -34,6 +35,10 @@ func (p *servicePackage) NewClient(ctx context.Context, config map[string]any) ( + if endpoint := config[names.AttrEndpoint].(string); endpoint != "" { + o.BaseEndpoint = aws_sdkv2.String(endpoint) + } ++ o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { ++ _, err := stack.Deserialize.Remove("TF_AWS_RequestResponseLogger") ++ return err ++ }) + o.Retryer = conns.AddIsErrorRetryables(cfg.Retryer().(aws_sdkv2.RetryerV2), retry) + }), nil + } From 4160450876b6377180d28c8fbb3845a677d22961 Mon Sep 17 00:00:00 2001 From: Florian Stadler Date: Fri, 31 May 2024 17:07:13 +0200 Subject: [PATCH 4/8] Create Logging Middleware for Lambda service that does not log the lambda code archive When creating lambda functions and directly uploading the code, then the whole archive is being logged as a base64 encoded string as part of the HTTP request logger. In order to do so, multiple copies of the body are created in memory, which leads to memory bloating. This change fixes that by redacting the body in the logs for the Create/Update Lambda calls. --- examples/examples_nodejs_test.go | 63 +++++++ examples/parallel-lambdas/Pulumi.yaml | 2 +- examples/parallel-lambdas/create_lambda.sh | 23 --- examples/parallel-lambdas/index.ts | 5 +- ...iddleware-for-Lambda-service-that-do.patch | 158 ++++++++++++++++++ ...ove-HTTP-logger-from-Lambda-resource.patch | 29 ---- 6 files changed, 225 insertions(+), 55 deletions(-) delete mode 100755 examples/parallel-lambdas/create_lambda.sh create mode 100644 patches/0060-Create-Logging-Middleware-for-Lambda-service-that-do.patch delete mode 100644 patches/0060-Remove-HTTP-logger-from-Lambda-resource.patch diff --git a/examples/examples_nodejs_test.go b/examples/examples_nodejs_test.go index ee257b7d4e0..4cb84281cac 100644 --- a/examples/examples_nodejs_test.go +++ b/examples/examples_nodejs_test.go @@ -5,7 +5,9 @@ package examples import ( + "archive/zip" "bytes" + "crypto/rand" "encoding/json" "io" "os" @@ -658,3 +660,64 @@ func TestRdsGetEngineVersion(t *testing.T) { engineVersion := res.Outputs["vs"] require.NotEmpty(t, engineVersion.Value) } + +func TestParallelLambdas(t *testing.T) { + tempFile, err := createLambdaArchive(25 * 1024 * 1024) + require.NoError(t, err) + defer os.Remove(tempFile) + test := getJSBaseOptions(t). + With(integration.ProgramTestOptions{ + Dir: filepath.Join(getCwd(t), "parallel-lambdas"), + UpdateCommandlineFlags: []string{"-v11", "--logtostderr", "--debug", "--logflow", "--parallel", "16"}, + // PreviewCommandlineFlags: []string{"--parallel", "1"}, + AllowEmptyPreviewChanges: true, + SkipRefresh: true, + Config: map[string]string{ + "lambda:archivePath": tempFile, + }, + }) + + integration.ProgramTest(t, &test) +} + +func createLambdaArchive(size int64) (string, error) { + // Create a temporary file to save the zip archive + tempFile, err := os.CreateTemp("", "archive-*.zip") + if err != nil { + return "", err + } + defer tempFile.Close() + + // Create a new zip archive + zipWriter := zip.NewWriter(tempFile) + defer zipWriter.Close() + + randomDataReader := io.LimitReader(rand.Reader, size) + + // Create the index.js file for the lambda + indexWriter, err := zipWriter.Create("index.js") + if err != nil { + return "", err + } + _, err = indexWriter.Write([]byte("const { version } = require(\"@aws-sdk/client-s3/package.json\");\n\nexports.handler = async () => ({ version });\n")) + if err != nil { + return "", err + } + + randomDataWriter, err := zipWriter.Create("random.txt") + if err != nil { + return "", err + } + _, err = io.Copy(randomDataWriter, randomDataReader) + if err != nil { + return "", err + } + + // Get the path of the temporary file + archivePath, err := filepath.Abs(tempFile.Name()) + if err != nil { + return "", err + } + + return archivePath, nil +} diff --git a/examples/parallel-lambdas/Pulumi.yaml b/examples/parallel-lambdas/Pulumi.yaml index 372cae9dd9b..1eafbd4f9d0 100644 --- a/examples/parallel-lambdas/Pulumi.yaml +++ b/examples/parallel-lambdas/Pulumi.yaml @@ -1,3 +1,3 @@ -name: kms-alias +name: parallel-lambdas runtime: nodejs description: Parallel Lambdas example diff --git a/examples/parallel-lambdas/create_lambda.sh b/examples/parallel-lambdas/create_lambda.sh deleted file mode 100755 index b40e79ea62c..00000000000 --- a/examples/parallel-lambdas/create_lambda.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -temp_dir=$(mktemp -d) -trap "rm -rf $temp_dir" EXIT -pushd $temp_dir - -printf "const { version } = require(\"@aws-sdk/client-s3/package.json\");\n\nexports.handler = async () => ({ version });\n" > index.js -dd if=/dev/urandom of=example.txt bs=25m count=1 -zip lambda.zip index.js example.txt - -# Check the size of lambda.zip -zip_size=$(stat -f %z lambda.zip) - -# Compare the size with 25MB (25 * 1024 * 1024 bytes) -if [[ $zip_size -lt $((25 * 1024 * 1024)) ]]; then - echo "lambda.zip is smaller than 25MB. Actual size $zip_size." - exit 1 -fi - -popd -mv $temp_dir/lambda.zip . diff --git a/examples/parallel-lambdas/index.ts b/examples/parallel-lambdas/index.ts index e302fec1db4..9c7ba89375e 100644 --- a/examples/parallel-lambdas/index.ts +++ b/examples/parallel-lambdas/index.ts @@ -17,6 +17,7 @@ import * as aws from "@pulumi/aws"; const config = new pulumi.Config("aws"); const providerOpts = { provider: new aws.Provider("prov", { region: config.require("envRegion") }) }; +const lambdaConfig = new pulumi.Config("lambda"); const assumeRole = aws.iam.getPolicyDocument({ statements: [{ @@ -32,9 +33,9 @@ const role = new aws.iam.Role("parallel-iam-role", { assumeRolePolicy: assumeRole.then(assumeRole => assumeRole.json) }, providerOpts); -for (let i = 0; i < 50; i++) { +for (let i = 0; i < 25; i++) { new aws.lambda.Function(`lambda-${i}`, { - code: new pulumi.asset.FileArchive("lambda.zip"), + code: new pulumi.asset.FileArchive(lambdaConfig.require("archivePath")), role: role.arn, handler: "index.handler", runtime: "nodejs20.x", diff --git a/patches/0060-Create-Logging-Middleware-for-Lambda-service-that-do.patch b/patches/0060-Create-Logging-Middleware-for-Lambda-service-that-do.patch new file mode 100644 index 00000000000..5cc50bfa97b --- /dev/null +++ b/patches/0060-Create-Logging-Middleware-for-Lambda-service-that-do.patch @@ -0,0 +1,158 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Florian Stadler +Date: Fri, 31 May 2024 12:29:36 +0200 +Subject: [PATCH] Create Logging Middleware for Lambda service that does not + log the lambda code archive + +When creating lambda functions and directly uploading the code, then the whole archive +is being logged as a base64 encoded string as part of the HTTP request logger. +In order to do so, multiple copies of the body are created in memory, which leads +to memory bloating. +This change fixes that by redacting the body in the logs for the Create/Update Lambda +calls. + +diff --git a/internal/service/lambda/request_response_logger.go b/internal/service/lambda/request_response_logger.go +new file mode 100644 +index 0000000000..58563221ab +--- /dev/null ++++ b/internal/service/lambda/request_response_logger.go +@@ -0,0 +1,107 @@ ++package lambda ++ ++import ( ++ "context" ++ "fmt" ++ "github.com/aws/aws-sdk-go-v2/aws" ++ awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" ++ "github.com/aws/smithy-go/middleware" ++ smithyhttp "github.com/aws/smithy-go/transport/http" ++ "github.com/hashicorp/aws-sdk-go-base/v2/logging" ++ "io" ++ "net/http" ++ "strings" ++ "time" ++ _ "unsafe" ++) ++ ++const ( ++ lambdaCreateOperation = "CreateFunction" ++ lambdaUpdateFunctionCodeOperation = "UpdateFunctionCode" ++) ++ ++// Replaces the upstream logging middleware from https://github.com/hashicorp/aws-sdk-go-base/blob/main/logger.go#L107 ++// We do not want to log the Lambda Archive that is part of the request body because this leads to bloating memory ++type wrappedRequestResponseLogger struct { ++ wrapped middleware.DeserializeMiddleware ++} ++ ++// ID is the middleware identifier. ++func (r *wrappedRequestResponseLogger) ID() string { ++ return "PULUMI_AWS_RequestResponseLogger" ++} ++ ++func NewWrappedRequestResponseLogger(wrapped middleware.DeserializeMiddleware) middleware.DeserializeMiddleware { ++ return &wrappedRequestResponseLogger{wrapped: wrapped} ++} ++ ++//go:linkname decomposeHTTPResponse github.com/hashicorp/aws-sdk-go-base/v2.decomposeHTTPResponse ++func decomposeHTTPResponse(ctx context.Context, resp *http.Response, elapsed time.Duration) (map[string]any, error) ++ ++func (r *wrappedRequestResponseLogger) HandleDeserialize(ctx context.Context, in middleware.DeserializeInput, next middleware.DeserializeHandler, ++) ( ++ out middleware.DeserializeOutput, metadata middleware.Metadata, err error, ++) { ++ if awsmiddleware.GetServiceID(ctx) == "Lambda" { ++ if op := awsmiddleware.GetOperationName(ctx); op != lambdaCreateOperation && op != lambdaUpdateFunctionCodeOperation { ++ // pass through to the wrapped response logger for all other lambda operations that do not send the code as part of the request body ++ return r.wrapped.HandleDeserialize(ctx, in, next) ++ } ++ } ++ ++ logger := logging.RetrieveLogger(ctx) ++ region := awsmiddleware.GetRegion(ctx) ++ ++ if signingRegion := awsmiddleware.GetSigningRegion(ctx); signingRegion != region { //nolint:staticcheck // Not retrievable elsewhere ++ ctx = logger.SetField(ctx, string(logging.SigningRegionKey), signingRegion) ++ } ++ if awsmiddleware.GetEndpointSource(ctx) == aws.EndpointSourceCustom { ++ ctx = logger.SetField(ctx, string(logging.CustomEndpointKey), true) ++ } ++ ++ req, ok := in.Request.(*smithyhttp.Request) ++ if !ok { ++ return out, metadata, fmt.Errorf("unexpected request middleware type %T", in.Request) ++ } ++ ++ rc := req.Build(ctx) ++ ++ originalBody := rc.Body ++ // remove the body from the logging output ++ redactedBody := strings.NewReader("[Redacted]") ++ rc.Body = io.NopCloser(redactedBody) ++ rc.ContentLength = redactedBody.Size() ++ ++ requestFields, err := logging.DecomposeHTTPRequest(ctx, rc) ++ if err != nil { ++ return out, metadata, fmt.Errorf("decomposing request: %w", err) ++ } ++ logger.Debug(ctx, "HTTP Request Sent", requestFields) ++ ++ // reconstruct the original request ++ req, err = req.SetStream(originalBody) ++ if err != nil { ++ return out, metadata, err ++ } ++ in.Request = req ++ ++ start := time.Now() ++ out, metadata, err = next.HandleDeserialize(ctx, in) ++ duration := time.Since(start) ++ ++ if err != nil { ++ return out, metadata, err ++ } ++ ++ if res, ok := out.RawResponse.(*smithyhttp.Response); !ok { ++ return out, metadata, fmt.Errorf("unknown response type: %T", out.RawResponse) ++ } else { ++ responseFields, err := decomposeHTTPResponse(ctx, res.Response, duration) ++ if err != nil { ++ return out, metadata, fmt.Errorf("decomposing response: %w", err) ++ } ++ logger.Debug(ctx, "HTTP Response Received", responseFields) ++ } ++ ++ return out, metadata, err ++} +diff --git a/internal/service/lambda/service_package_extra.go b/internal/service/lambda/service_package_extra.go +index 54f6aac15a..1f2440d3e3 100644 +--- a/internal/service/lambda/service_package_extra.go ++++ b/internal/service/lambda/service_package_extra.go +@@ -6,6 +6,7 @@ import ( + aws_sdkv2 "github.com/aws/aws-sdk-go-v2/aws" + retry_sdkv2 "github.com/aws/aws-sdk-go-v2/aws/retry" + lambda_sdkv2 "github.com/aws/aws-sdk-go-v2/service/lambda" ++ "github.com/aws/smithy-go/middleware" + tfawserr_sdkv2 "github.com/hashicorp/aws-sdk-go-base/v2/tfawserr" + "github.com/hashicorp/terraform-provider-aws/internal/conns" + "github.com/hashicorp/terraform-provider-aws/names" +@@ -34,6 +35,19 @@ func (p *servicePackage) NewClient(ctx context.Context, config map[string]any) ( + if endpoint := config[names.AttrEndpoint].(string); endpoint != "" { + o.BaseEndpoint = aws_sdkv2.String(endpoint) + } ++ ++ // Switch out the terraform http logging middleware with a custom logging middleware that does not log the ++ // lambda code. Logging the lambda code leads to memory bloating because it allocates a lot of copies of the ++ // body ++ o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { ++ loggingMiddleware, err := stack.Deserialize.Remove("TF_AWS_RequestResponseLogger") ++ if err != nil { ++ return err ++ } ++ ++ err = stack.Deserialize.Add(NewWrappedRequestResponseLogger(loggingMiddleware), middleware.After) ++ return err ++ }) + o.Retryer = conns.AddIsErrorRetryables(cfg.Retryer().(aws_sdkv2.RetryerV2), retry) + }), nil + } diff --git a/patches/0060-Remove-HTTP-logger-from-Lambda-resource.patch b/patches/0060-Remove-HTTP-logger-from-Lambda-resource.patch deleted file mode 100644 index 768431dc82f..00000000000 --- a/patches/0060-Remove-HTTP-logger-from-Lambda-resource.patch +++ /dev/null @@ -1,29 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Florian Stadler -Date: Fri, 24 May 2024 09:44:07 +0200 -Subject: [PATCH] Remove HTTP logger from Lambda resource - - -diff --git a/internal/service/lambda/service_package_extra.go b/internal/service/lambda/service_package_extra.go -index 54f6aac15a..daa656b829 100644 ---- a/internal/service/lambda/service_package_extra.go -+++ b/internal/service/lambda/service_package_extra.go -@@ -6,6 +6,7 @@ import ( - aws_sdkv2 "github.com/aws/aws-sdk-go-v2/aws" - retry_sdkv2 "github.com/aws/aws-sdk-go-v2/aws/retry" - lambda_sdkv2 "github.com/aws/aws-sdk-go-v2/service/lambda" -+ "github.com/aws/smithy-go/middleware" - tfawserr_sdkv2 "github.com/hashicorp/aws-sdk-go-base/v2/tfawserr" - "github.com/hashicorp/terraform-provider-aws/internal/conns" - "github.com/hashicorp/terraform-provider-aws/names" -@@ -34,6 +35,10 @@ func (p *servicePackage) NewClient(ctx context.Context, config map[string]any) ( - if endpoint := config[names.AttrEndpoint].(string); endpoint != "" { - o.BaseEndpoint = aws_sdkv2.String(endpoint) - } -+ o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { -+ _, err := stack.Deserialize.Remove("TF_AWS_RequestResponseLogger") -+ return err -+ }) - o.Retryer = conns.AddIsErrorRetryables(cfg.Retryer().(aws_sdkv2.RetryerV2), retry) - }), nil - } From 9542654c67304bd84375f1a8289934e0d9c63d66 Mon Sep 17 00:00:00 2001 From: Florian Stadler Date: Fri, 31 May 2024 17:14:58 +0200 Subject: [PATCH 5/8] remove debugging leftovers --- examples/parallel-lambdas-s3/Pulumi.yaml | 3 - examples/parallel-lambdas-s3/create_lambda.sh | 23 ------- examples/parallel-lambdas-s3/index.ts | 68 ------------------- examples/parallel-lambdas-s3/package.json | 16 ----- examples/parallel-lambdas-s3/tsconfig.json | 18 ----- 5 files changed, 128 deletions(-) delete mode 100644 examples/parallel-lambdas-s3/Pulumi.yaml delete mode 100755 examples/parallel-lambdas-s3/create_lambda.sh delete mode 100644 examples/parallel-lambdas-s3/index.ts delete mode 100644 examples/parallel-lambdas-s3/package.json delete mode 100644 examples/parallel-lambdas-s3/tsconfig.json diff --git a/examples/parallel-lambdas-s3/Pulumi.yaml b/examples/parallel-lambdas-s3/Pulumi.yaml deleted file mode 100644 index 372cae9dd9b..00000000000 --- a/examples/parallel-lambdas-s3/Pulumi.yaml +++ /dev/null @@ -1,3 +0,0 @@ -name: kms-alias -runtime: nodejs -description: Parallel Lambdas example diff --git a/examples/parallel-lambdas-s3/create_lambda.sh b/examples/parallel-lambdas-s3/create_lambda.sh deleted file mode 100755 index b40e79ea62c..00000000000 --- a/examples/parallel-lambdas-s3/create_lambda.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -temp_dir=$(mktemp -d) -trap "rm -rf $temp_dir" EXIT -pushd $temp_dir - -printf "const { version } = require(\"@aws-sdk/client-s3/package.json\");\n\nexports.handler = async () => ({ version });\n" > index.js -dd if=/dev/urandom of=example.txt bs=25m count=1 -zip lambda.zip index.js example.txt - -# Check the size of lambda.zip -zip_size=$(stat -f %z lambda.zip) - -# Compare the size with 25MB (25 * 1024 * 1024 bytes) -if [[ $zip_size -lt $((25 * 1024 * 1024)) ]]; then - echo "lambda.zip is smaller than 25MB. Actual size $zip_size." - exit 1 -fi - -popd -mv $temp_dir/lambda.zip . diff --git a/examples/parallel-lambdas-s3/index.ts b/examples/parallel-lambdas-s3/index.ts deleted file mode 100644 index cb29625ed17..00000000000 --- a/examples/parallel-lambdas-s3/index.ts +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2016-2018, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import * as pulumi from "@pulumi/pulumi"; -import * as aws from "@pulumi/aws"; -import * as std from "@pulumi/std"; - -const config = new pulumi.Config("aws"); -const providerOpts = { provider: new aws.Provider("prov", { region: config.require("envRegion") }) }; - -const assumeRole = aws.iam.getPolicyDocument({ - statements: [{ - effect: "Allow", - principals: [{ - type: "Service", - identifiers: ["lambda.amazonaws.com"], - }], - actions: ["sts:AssumeRole"], - }], -}, providerOpts); -const role = new aws.iam.Role("parallel-iam-role", { - assumeRolePolicy: assumeRole.then(assumeRole => assumeRole.json) -}, providerOpts); - -const bucket = new aws.s3.BucketV2("lambda-code-storage", { - forceDestroy: true, -}, providerOpts); - -const versioning = new aws.s3.BucketVersioningV2("lambda-code-storage", { - bucket: bucket.id, - versioningConfiguration: { - status: "Enabled", - }, -}, providerOpts); - -for (let i = 0; i < 50; i++) { - const lambdaCode = new aws.s3.BucketObjectv2(`lambda-zip-${i}`, { - bucket: bucket.bucket, - key: `lambda-${i}.zip`, - source: new pulumi.asset.FileAsset("lambda.zip"), - sourceHash: std.filesha256({ - input: "lambda.zip", - }).then(invoke => invoke.result), - }, {...providerOpts, dependsOn: [versioning]}); - - new aws.lambda.Function(`lambda-${i}`, { - role: role.arn, - handler: "index.handler", - runtime: "nodejs20.x", - s3Bucket: bucket.bucket, - s3Key: lambdaCode.key, - s3ObjectVersion: lambdaCode.versionId, - }, { - ...providerOpts, - ignoreChanges: ["replacementSecurityGroupIds"], - }); -} diff --git a/examples/parallel-lambdas-s3/package.json b/examples/parallel-lambdas-s3/package.json deleted file mode 100644 index b88deddba3d..00000000000 --- a/examples/parallel-lambdas-s3/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "parallel-lambdas-s3", - "version": "0.0.1", - "license": "Apache-2.0", - "scripts": { - "build": "tsc" - }, - "dependencies": { - "@pulumi/pulumi": "^3.0.0", - "@pulumi/aws": "^6.0.0", - "@pulumi/std": "^1.6.2" - }, - "devDependencies": { - "@types/node": "^8.0.0" - } -} diff --git a/examples/parallel-lambdas-s3/tsconfig.json b/examples/parallel-lambdas-s3/tsconfig.json deleted file mode 100644 index ab65afa6135..00000000000 --- a/examples/parallel-lambdas-s3/tsconfig.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "compilerOptions": { - "strict": true, - "outDir": "bin", - "target": "es2016", - "module": "commonjs", - "moduleResolution": "node", - "sourceMap": true, - "experimentalDecorators": true, - "pretty": true, - "noFallthroughCasesInSwitch": true, - "noImplicitReturns": true, - "forceConsistentCasingInFileNames": true - }, - "files": [ - "index.ts" - ] -} From 32751c5753085db71ad973c014955ff2dd57f6dc Mon Sep 17 00:00:00 2001 From: Florian Stadler Date: Fri, 31 May 2024 17:28:28 +0200 Subject: [PATCH 6/8] Move test to provider tests --- examples/examples_nodejs_test.go | 63 ----------------- provider/provider_nodejs_test.go | 69 +++++++++++++++++++ .../parallel-lambdas/Pulumi.yaml | 0 .../test-programs}/parallel-lambdas/index.ts | 0 .../parallel-lambdas/package.json | 0 .../parallel-lambdas/tsconfig.json | 0 6 files changed, 69 insertions(+), 63 deletions(-) rename {examples => provider/test-programs}/parallel-lambdas/Pulumi.yaml (100%) rename {examples => provider/test-programs}/parallel-lambdas/index.ts (100%) rename {examples => provider/test-programs}/parallel-lambdas/package.json (100%) rename {examples => provider/test-programs}/parallel-lambdas/tsconfig.json (100%) diff --git a/examples/examples_nodejs_test.go b/examples/examples_nodejs_test.go index 4cb84281cac..ee257b7d4e0 100644 --- a/examples/examples_nodejs_test.go +++ b/examples/examples_nodejs_test.go @@ -5,9 +5,7 @@ package examples import ( - "archive/zip" "bytes" - "crypto/rand" "encoding/json" "io" "os" @@ -660,64 +658,3 @@ func TestRdsGetEngineVersion(t *testing.T) { engineVersion := res.Outputs["vs"] require.NotEmpty(t, engineVersion.Value) } - -func TestParallelLambdas(t *testing.T) { - tempFile, err := createLambdaArchive(25 * 1024 * 1024) - require.NoError(t, err) - defer os.Remove(tempFile) - test := getJSBaseOptions(t). - With(integration.ProgramTestOptions{ - Dir: filepath.Join(getCwd(t), "parallel-lambdas"), - UpdateCommandlineFlags: []string{"-v11", "--logtostderr", "--debug", "--logflow", "--parallel", "16"}, - // PreviewCommandlineFlags: []string{"--parallel", "1"}, - AllowEmptyPreviewChanges: true, - SkipRefresh: true, - Config: map[string]string{ - "lambda:archivePath": tempFile, - }, - }) - - integration.ProgramTest(t, &test) -} - -func createLambdaArchive(size int64) (string, error) { - // Create a temporary file to save the zip archive - tempFile, err := os.CreateTemp("", "archive-*.zip") - if err != nil { - return "", err - } - defer tempFile.Close() - - // Create a new zip archive - zipWriter := zip.NewWriter(tempFile) - defer zipWriter.Close() - - randomDataReader := io.LimitReader(rand.Reader, size) - - // Create the index.js file for the lambda - indexWriter, err := zipWriter.Create("index.js") - if err != nil { - return "", err - } - _, err = indexWriter.Write([]byte("const { version } = require(\"@aws-sdk/client-s3/package.json\");\n\nexports.handler = async () => ({ version });\n")) - if err != nil { - return "", err - } - - randomDataWriter, err := zipWriter.Create("random.txt") - if err != nil { - return "", err - } - _, err = io.Copy(randomDataWriter, randomDataReader) - if err != nil { - return "", err - } - - // Get the path of the temporary file - archivePath, err := filepath.Abs(tempFile.Name()) - if err != nil { - return "", err - } - - return archivePath, nil -} diff --git a/provider/provider_nodejs_test.go b/provider/provider_nodejs_test.go index 01757d9de42..502bfd39fc8 100644 --- a/provider/provider_nodejs_test.go +++ b/provider/provider_nodejs_test.go @@ -94,3 +94,72 @@ func TestRegressAttributeMustBeWholeNumber(t *testing.T) { result := test.Preview() t.Logf("#%v", result.ChangeSummary) } + +func TestParallelLambdaCreation(t *testing.T) { + if testing.Short() { + t.Skipf("Skipping test in -short mode because it needs cloud credentials") + return + } + + tempFile, err := createLambdaArchive(25 * 1024 * 1024) + require.NoError(t, err) + defer os.Remove(tempFile) + + maxDuration(5*time.Minute, t, func(t *testing.T) { + test := getJSBaseOptions(t). + With(integration.ProgramTestOptions{ + Dir: filepath.Join("test-programs", "parallel-lambdas"), + Config: map[string]string{ + "lambda:archivePath": tempFile, + }, + // Lambdas have diffs on every update (source code hash) + AllowEmptyPreviewChanges: true, + SkipRefresh: true, + }) + + integration.ProgramTest(t, &test) + } +} + +func createLambdaArchive(size int64) (string, error) { + // Create a temporary file to save the zip archive + tempFile, err := os.CreateTemp("", "archive-*.zip") + if err != nil { + return "", err + } + defer tempFile.Close() + + // Create a new zip archive + zipWriter := zip.NewWriter(tempFile) + defer zipWriter.Close() + + randomDataReader := io.LimitReader(rand.Reader, size) + + // Create the index.js file for the lambda + indexWriter, err := zipWriter.Create("index.js") + if err != nil { + return "", err + } + _, err = indexWriter.Write([]byte("const { version } = require(\"@aws-sdk/client-s3/package.json\");\n\nexports.handler = async () => ({ version });\n")) + if err != nil { + return "", err + } + + randomDataWriter, err := zipWriter.Create("random.txt") + if err != nil { + return "", err + } + _, err = io.Copy(randomDataWriter, randomDataReader) + if err != nil { + return "", err + } + + // Get the path of the temporary file + archivePath, err := filepath.Abs(tempFile.Name()) + if err != nil { + return "", err + } + + return archivePath, nil +} + diff --git a/examples/parallel-lambdas/Pulumi.yaml b/provider/test-programs/parallel-lambdas/Pulumi.yaml similarity index 100% rename from examples/parallel-lambdas/Pulumi.yaml rename to provider/test-programs/parallel-lambdas/Pulumi.yaml diff --git a/examples/parallel-lambdas/index.ts b/provider/test-programs/parallel-lambdas/index.ts similarity index 100% rename from examples/parallel-lambdas/index.ts rename to provider/test-programs/parallel-lambdas/index.ts diff --git a/examples/parallel-lambdas/package.json b/provider/test-programs/parallel-lambdas/package.json similarity index 100% rename from examples/parallel-lambdas/package.json rename to provider/test-programs/parallel-lambdas/package.json diff --git a/examples/parallel-lambdas/tsconfig.json b/provider/test-programs/parallel-lambdas/tsconfig.json similarity index 100% rename from examples/parallel-lambdas/tsconfig.json rename to provider/test-programs/parallel-lambdas/tsconfig.json From c3d0eba8e8e725746bf8084f8f517e67b1055ab1 Mon Sep 17 00:00:00 2001 From: Florian Stadler Date: Fri, 31 May 2024 18:36:11 +0200 Subject: [PATCH 7/8] Fix syntax error in test --- provider/provider_nodejs_test.go | 45 +++++++++++++++++++++++--------- provider/provider_python_test.go | 15 ----------- provider/provider_test.go | 16 ++++++++++++ 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/provider/provider_nodejs_test.go b/provider/provider_nodejs_test.go index 502bfd39fc8..85ec4f944c9 100644 --- a/provider/provider_nodejs_test.go +++ b/provider/provider_nodejs_test.go @@ -6,12 +6,17 @@ package provider import ( + "archive/zip" + "crypto/rand" + "io" "os" "path/filepath" "testing" + "time" "github.com/pulumi/providertest/pulumitest" "github.com/pulumi/providertest/pulumitest/opttest" + "github.com/pulumi/pulumi/pkg/v3/testing/integration" "github.com/stretchr/testify/require" ) @@ -100,25 +105,40 @@ func TestParallelLambdaCreation(t *testing.T) { t.Skipf("Skipping test in -short mode because it needs cloud credentials") return } - + tempFile, err := createLambdaArchive(25 * 1024 * 1024) require.NoError(t, err) defer os.Remove(tempFile) maxDuration(5*time.Minute, t, func(t *testing.T) { test := getJSBaseOptions(t). - With(integration.ProgramTestOptions{ - Dir: filepath.Join("test-programs", "parallel-lambdas"), - Config: map[string]string{ - "lambda:archivePath": tempFile, - }, - // Lambdas have diffs on every update (source code hash) - AllowEmptyPreviewChanges: true, - SkipRefresh: true, - }) - - integration.ProgramTest(t, &test) + With(integration.ProgramTestOptions{ + Dir: filepath.Join("test-programs", "parallel-lambdas"), + Config: map[string]string{ + "lambda:archivePath": tempFile, + }, + // Lambdas have diffs on every update (source code hash) + AllowEmptyPreviewChanges: true, + SkipRefresh: true, + }) + + integration.ProgramTest(t, &test) + }) +} + +func getJSBaseOptions(t *testing.T) integration.ProgramTestOptions { + envRegion := getEnvRegion(t) + baseJS := integration.ProgramTestOptions{ + Config: map[string]string{ + "aws:region": "INVALID_REGION", + "aws:envRegion": envRegion, + }, + Dependencies: []string{ + "@pulumi/aws", + }, } + + return baseJS } func createLambdaArchive(size int64) (string, error) { @@ -162,4 +182,3 @@ func createLambdaArchive(size int64) (string, error) { return archivePath, nil } - diff --git a/provider/provider_python_test.go b/provider/provider_python_test.go index 82dbb41c764..353c7acff4e 100644 --- a/provider/provider_python_test.go +++ b/provider/provider_python_test.go @@ -105,18 +105,3 @@ func getPythonBaseOptions(t *testing.T) integration.ProgramTestOptions { return pythonBase } - -func maxDuration(dur time.Duration, t *testing.T, test func(t *testing.T)) { - t.Helper() - timeout := time.After(dur) - done := make(chan bool) - go func() { - test(t) - done <- true - }() - select { - case <-timeout: - t.Fatalf("Test timed out after %v", dur) - case <-done: - } -} diff --git a/provider/provider_test.go b/provider/provider_test.go index 1589aa4d10a..8a75d6f4537 100644 --- a/provider/provider_test.go +++ b/provider/provider_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/pulumi/providertest" "github.com/pulumi/providertest/optproviderupgrade" @@ -86,3 +87,18 @@ func pulumiTest(t *testing.T, dir string, opts ...opttest.Option) *pulumitest.Pu ptest := pulumitest.NewPulumiTest(t, dir, opts...) return ptest } + +func maxDuration(dur time.Duration, t *testing.T, test func(t *testing.T)) { + t.Helper() + timeout := time.After(dur) + done := make(chan bool) + go func() { + test(t) + done <- true + }() + select { + case <-timeout: + t.Fatalf("Test timed out after %v", dur) + case <-done: + } +} From a47b2cbf3b6fab300e9735cf6ae578e6ae0a85e9 Mon Sep 17 00:00:00 2001 From: Florian Stadler Date: Fri, 7 Jun 2024 00:20:16 +0200 Subject: [PATCH 8/8] Rename patches after merging in master --- ...Parallelize-Lambda-Function-resource-operations.patch} | 8 ++++---- ...e-Logging-Middleware-for-Lambda-service-that-do.patch} | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) rename patches/{0059-Parallelize-Lambda-Function-resource-operations.patch => 0060-Parallelize-Lambda-Function-resource-operations.patch} (89%) rename patches/{0060-Create-Logging-Middleware-for-Lambda-service-that-do.patch => 0061-Create-Logging-Middleware-for-Lambda-service-that-do.patch} (95%) diff --git a/patches/0059-Parallelize-Lambda-Function-resource-operations.patch b/patches/0060-Parallelize-Lambda-Function-resource-operations.patch similarity index 89% rename from patches/0059-Parallelize-Lambda-Function-resource-operations.patch rename to patches/0060-Parallelize-Lambda-Function-resource-operations.patch index 165dd855708..fa2a578619f 100644 --- a/patches/0059-Parallelize-Lambda-Function-resource-operations.patch +++ b/patches/0060-Parallelize-Lambda-Function-resource-operations.patch @@ -10,10 +10,10 @@ We think this was an optimization for a special edge case that drastically worsens the UX for the majority of users. diff --git a/internal/service/lambda/function.go b/internal/service/lambda/function.go -index 4c49c5fb21..f2ce387f67 100644 +index fb1d412f18..8e3529cc26 100644 --- a/internal/service/lambda/function.go +++ b/internal/service/lambda/function.go -@@ -35,7 +35,6 @@ import ( +@@ -36,7 +36,6 @@ import ( const ( FunctionVersionLatest = "$LATEST" @@ -21,7 +21,7 @@ index 4c49c5fb21..f2ce387f67 100644 listVersionsMaxItems = 10000 ) -@@ -481,11 +480,6 @@ func resourceFunctionCreate(ctx context.Context, d *schema.ResourceData, meta in +@@ -482,11 +481,6 @@ func resourceFunctionCreate(ctx context.Context, d *schema.ResourceData, meta in } if v, ok := d.GetOk("filename"); ok { @@ -33,7 +33,7 @@ index 4c49c5fb21..f2ce387f67 100644 zipFile, err := readFileContents(v.(string)) if err != nil { -@@ -942,11 +936,6 @@ func resourceFunctionUpdate(ctx context.Context, d *schema.ResourceData, meta in +@@ -944,11 +938,6 @@ func resourceFunctionUpdate(ctx context.Context, d *schema.ResourceData, meta in } if v, ok := d.GetOk("filename"); ok { diff --git a/patches/0060-Create-Logging-Middleware-for-Lambda-service-that-do.patch b/patches/0061-Create-Logging-Middleware-for-Lambda-service-that-do.patch similarity index 95% rename from patches/0060-Create-Logging-Middleware-for-Lambda-service-that-do.patch rename to patches/0061-Create-Logging-Middleware-for-Lambda-service-that-do.patch index 5cc50bfa97b..af7a069e2ee 100644 --- a/patches/0060-Create-Logging-Middleware-for-Lambda-service-that-do.patch +++ b/patches/0061-Create-Logging-Middleware-for-Lambda-service-that-do.patch @@ -13,10 +13,10 @@ calls. diff --git a/internal/service/lambda/request_response_logger.go b/internal/service/lambda/request_response_logger.go new file mode 100644 -index 0000000000..58563221ab +index 0000000000..737faef4a7 --- /dev/null +++ b/internal/service/lambda/request_response_logger.go -@@ -0,0 +1,107 @@ +@@ -0,0 +1,109 @@ +package lambda + +import ( @@ -68,6 +68,8 @@ index 0000000000..58563221ab + } + } + ++ // Inlined the logging middleware from https://github.com/hashicorp/aws-sdk-go-base/blob/main/logger.go and patching ++ // out the request body logging + logger := logging.RetrieveLogger(ctx) + region := awsmiddleware.GetRegion(ctx) + @@ -86,7 +88,7 @@ index 0000000000..58563221ab + rc := req.Build(ctx) + + originalBody := rc.Body -+ // remove the body from the logging output ++ // remove the body from the logging output. This is the main change compared to the upstream logging middleware + redactedBody := strings.NewReader("[Redacted]") + rc.Body = io.NopCloser(redactedBody) + rc.ContentLength = redactedBody.Size()