fix(integ): fix rate limit errors when deploying cloudwatch log group…

integ/components/deadline/deadline_01_repository/bin/deadline_01_repository.ts

@@ -8,6 +8,7 @@ import {
  Stage,
  ThinkboxDockerRecipes,
 } from 'aws-rfdk/deadline';
+import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect';
 
 import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect';
 import { DatabaseType, StorageStruct } from '../../../../lib/storage-struct';
@@ -50,3 +51,5 @@ new RepositoryTestingTier(app, 'RFDKInteg-DL-TestingTier' + integStackTag, { env
 
 // Adds IAM Policy to Instance and ASG Roles
 Aspects.of(app).add(new SSMInstancePolicyAspect());
+// Adds log retention retry to all functions
+Aspects.of(app).add(new LogRetentionRetryAspect());

integ/components/deadline/deadline_02_renderQueue/bin/deadline_02_renderQueue.ts

@@ -8,6 +8,7 @@ import {
  Stage,
  ThinkboxDockerRecipes,
 } from 'aws-rfdk/deadline';
+import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect';
 
 import { RenderStruct } from '../../../../lib/render-struct';
 import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect';
@@ -61,3 +62,5 @@ new RenderQueueTestingTier(app, 'RFDKInteg-RQ-TestingTier' + integStackTag, { en
 
 // Adds IAM Policy to Instance and ASG Roles
 Aspects.of(app).add(new SSMInstancePolicyAspect());
+// Adds log retention retry to all functions
+Aspects.of(app).add(new LogRetentionRetryAspect());

integ/components/deadline/deadline_03_workerFleetHttp/bin/deadline_03_workerFleetHttp.ts

@@ -8,6 +8,7 @@ import {
  Stage,
  ThinkboxDockerRecipes,
 } from 'aws-rfdk/deadline';
+import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect';
 
 import { RenderStruct } from '../../../../lib/render-struct';
 import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect';
@@ -65,3 +66,5 @@ new WorkerFleetTestingTier(app, 'RFDKInteg-WF-TestingTier' + integStackTag, {env
 
 // Adds IAM Policy to Instance and ASG Roles
 Aspects.of(app).add(new SSMInstancePolicyAspect());
+// Adds log retention retry to all functions
+Aspects.of(app).add(new LogRetentionRetryAspect());

integ/components/deadline/deadline_04_workerFleetHttps/bin/deadline_04_workerFleetHttps.ts

@@ -8,6 +8,7 @@ import {
  Stage,
  ThinkboxDockerRecipes,
 } from 'aws-rfdk/deadline';
+import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect';
 
 import { RenderStruct } from '../../../../lib/render-struct';
 import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect';
@@ -65,3 +66,5 @@ new WorkerFleetTestingTier(app, 'RFDKInteg-WFS-TestingTier' + integStackTag, {en
 
 // Adds IAM Policy to Instance and ASG Roles
 Aspects.of(app).add(new SSMInstancePolicyAspect());
+// Adds log retention retry to all functions
+Aspects.of(app).add(new LogRetentionRetryAspect());

integ/components/deadline/deadline_05_secretsManagement/bin/deadline_05_secretsManagement.ts

@@ -12,6 +12,7 @@ import {
  ThinkboxDockerRecipes,
  UsageBasedLicense,
 } from 'aws-rfdk/deadline';
+import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect';
 import {
  RenderStruct,
  RenderStructUsageBasedLicensingProps,
@@ -79,6 +80,8 @@ async function main() {
 
  // Adds IAM Policy to Instance and ASG Roles
  Aspects.of(app).add(new SSMInstancePolicyAspect());
+ // Adds log retention retry to all functions
+ Aspects.of(app).add(new LogRetentionRetryAspect());
 }
 
 /**

integ/lib/log-retention-retry-aspect.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { CfnResource, IAspect, Stack } from 'aws-cdk-lib';
+import { CfnFunction } from 'aws-cdk-lib/aws-lambda';
+import { LogRetention } from 'aws-cdk-lib/aws-logs';
+import { IConstruct } from 'constructs';
+
+export class LogRetentionRetryAspect implements IAspect {
+ public visit(node: IConstruct): void {
+ // Define log retention retries to reduce the risk of the rate exceed error
+ // as the default create log group TPS is only 5. Make sure to set the timeout of log retention function
+ // to be greater than total retry time. That's because if the function that is used for a custom resource
+ // doesn't exit properly, it'd end up in retries and may take cloud formation an hour to realize that
+ // the custom resource failed.
+ if (node instanceof LogRetention) {
+ const logRetentionResource = node.node.defaultChild as CfnResource;
+ logRetentionResource.addPropertyOverride('SdkRetry', {
+ maxRetries: 7,
+ base: 200,
+ });
+
+ // referenced from cdk code: https://github.com/aws/aws-cdk/blob/v2.33.0/packages/@aws-cdk/aws-logs/lib/log-retention.ts#L116
+ const logRetentionFunctionConstructId = 'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a';
+ const logRetentionFunction = Stack.of(node).node.findChild(logRetentionFunctionConstructId);
+ const cfnFunction = logRetentionFunction.node.defaultChild as CfnFunction;
+ cfnFunction.addPropertyOverride('Timeout', 30);
+ }
+ }
+}