fix(cdk): changed retry mechanism for hotswapping AppSync.function (a…

…ws#32179)

### Reason for this change


Fixing bug in hotswap for `AppSync.function` where `ConcurrentModificationException` error continues to appear when attempting hotswap on a large collection of resources.
### Description of changes
Switches the retry mechanism for hotswapping AppSync.function from a simple retry where hotswap of AppSync.function is retried 5 times at 1 second intervals if a Concurrent Modification Exception occurs to an exponential back off retry, where the retry interval doubles with each failure. 
Exponential backoff will try 6 times (which through my testing should cover all reasonable cases) to resolve AppSync.function hotswap.


### Description of how you validated changes


No integration or unit tests added. Passes all current unit tests.
Tested locally on an Amplify project to ensure that cases that previously experienced `ConcurrentModificationException` errors no longer experienced them.
### Checklist
- [x] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md)

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/aws-cdk/lib/api/hotswap/appsync-mapping-templates.ts

@@ -118,13 +118,14 @@ export async function isHotswappableAppSyncChange(
           const functions = await sdk.appsync().listFunctions({ apiId: sdkRequestObject.apiId });
           const { functionId } = functions.find((fn) => fn.name === physicalName) ?? {};
           // Updating multiple functions at the same time or along with graphql schema results in `ConcurrentModificationException`
-          await simpleRetry(
+          await exponentialBackOffRetry(
             () =>
               sdk.appsync().updateFunction({
                 ...sdkRequestObject,
                 functionId: functionId,
               }),
-            5,
+            6,
+            1000,
             'ConcurrentModificationException',
           );
         } else if (isGraphQLSchema) {
@@ -169,13 +170,13 @@ async function fetchFileFromS3(s3Url: string, sdk: SDK) {
   return (await sdk.s3().getObject({ Bucket: s3Bucket, Key: s3Key })).Body?.transformToString();
 }
 
-async function simpleRetry(fn: () => Promise<any>, numOfRetries: number, errorCodeToRetry: string) {
+async function exponentialBackOffRetry(fn: () => Promise<any>, numOfRetries: number, backOff: number, errorCodeToRetry: string) {
   try {
     await fn();
   } catch (error: any) {
     if (error && error.name === errorCodeToRetry && numOfRetries > 0) {
-      await sleep(1000); // wait a whole second
-      await simpleRetry(fn, numOfRetries - 1, errorCodeToRetry);
+      await sleep(backOff); // time to wait doubles everytime function fails, starts at 1 second
+      await exponentialBackOffRetry(fn, numOfRetries - 1, backOff * 2, errorCodeToRetry);
     } else {
       throw error;
     }

packages/aws-cdk/test/api/hotswap/appsync-mapping-templates-hotswap-deployments.test.ts

@@ -969,6 +969,164 @@ describe.each([HotswapMode.FALL_BACK, HotswapMode.HOTSWAP_ONLY])('%p mode', (hot
     },
   );
 
+  silentTest(
+    'updateFunction() API recovers from failed update attempt through retry logic',
+    async () => {
+
+      // GIVEN
+      mockAppSyncClient
+        .on(ListFunctionsCommand)
+        .resolvesOnce({
+          functions: [{ name: 'my-function', functionId: 'functionId' }],
+        });
+
+      const ConcurrentModError = new Error('ConcurrentModificationException: Schema is currently being altered, please wait until that is complete.');
+      ConcurrentModError.name = 'ConcurrentModificationException';
+      mockAppSyncClient
+        .on(UpdateFunctionCommand)
+        .rejectsOnce(ConcurrentModError)
+        .resolvesOnce({ functionConfiguration: { name: 'my-function', dataSourceName: 'my-datasource', functionId: 'functionId' } });
+
+      setup.setCurrentCfnStackTemplate({
+        Resources: {
+          AppSyncFunction: {
+            Type: 'AWS::AppSync::FunctionConfiguration',
+            Properties: {
+              Name: 'my-function',
+              ApiId: 'apiId',
+              DataSourceName: 'my-datasource',
+              FunctionVersion: '2018-05-29',
+              RequestMappingTemplate: '## original request template',
+              ResponseMappingTemplate: '## original response template',
+            },
+            Metadata: {
+              'aws:asset:path': 'old-path',
+            },
+          },
+        },
+      });
+      const cdkStackArtifact = setup.cdkStackArtifactOf({
+        template: {
+          Resources: {
+            AppSyncFunction: {
+              Type: 'AWS::AppSync::FunctionConfiguration',
+              Properties: {
+                Name: 'my-function',
+                ApiId: 'apiId',
+                DataSourceName: 'my-datasource',
+                FunctionVersion: '2018-05-29',
+                RequestMappingTemplate: '## original request template',
+                ResponseMappingTemplate: '## new response template',
+              },
+              Metadata: {
+                'aws:asset:path': 'new-path',
+              },
+            },
+          },
+        },
+      });
+
+      // WHEN
+      const deployStackResult = await hotswapMockSdkProvider.tryHotswapDeployment(hotswapMode, cdkStackArtifact);
+
+      // THEN
+      expect(deployStackResult).not.toBeUndefined();
+      expect(mockAppSyncClient).toHaveReceivedCommandTimes(UpdateFunctionCommand, 2); // 1st failure then success on retry
+      expect(mockAppSyncClient).toHaveReceivedCommandWith(UpdateFunctionCommand, {
+        apiId: 'apiId',
+        dataSourceName: 'my-datasource',
+        functionId: 'functionId',
+        functionVersion: '2018-05-29',
+        name: 'my-function',
+        requestMappingTemplate: '## original request template',
+        responseMappingTemplate: '## new response template',
+      });
+    },
+  );
+
+  silentTest(
+    'updateFunction() API fails if it recieves 7 failed attempts in a row - this is a long running test',
+    async () => {
+
+      // GIVEN
+      mockAppSyncClient
+        .on(ListFunctionsCommand)
+        .resolvesOnce({
+          functions: [{ name: 'my-function', functionId: 'functionId' }],
+        });
+
+      const ConcurrentModError = new Error('ConcurrentModificationException: Schema is currently being altered, please wait until that is complete.');
+      ConcurrentModError.name = 'ConcurrentModificationException';
+      mockAppSyncClient
+        .on(UpdateFunctionCommand)
+        .rejectsOnce(ConcurrentModError)
+        .rejectsOnce(ConcurrentModError)
+        .rejectsOnce(ConcurrentModError)
+        .rejectsOnce(ConcurrentModError)
+        .rejectsOnce(ConcurrentModError)
+        .rejectsOnce(ConcurrentModError)
+        .rejectsOnce(ConcurrentModError)
+        .resolvesOnce({ functionConfiguration: { name: 'my-function', dataSourceName: 'my-datasource', functionId: 'functionId' } });
+
+      setup.setCurrentCfnStackTemplate({
+        Resources: {
+          AppSyncFunction: {
+            Type: 'AWS::AppSync::FunctionConfiguration',
+            Properties: {
+              Name: 'my-function',
+              ApiId: 'apiId',
+              DataSourceName: 'my-datasource',
+              FunctionVersion: '2018-05-29',
+              RequestMappingTemplate: '## original request template',
+              ResponseMappingTemplate: '## original response template',
+            },
+            Metadata: {
+              'aws:asset:path': 'old-path',
+            },
+          },
+        },
+      });
+      const cdkStackArtifact = setup.cdkStackArtifactOf({
+        template: {
+          Resources: {
+            AppSyncFunction: {
+              Type: 'AWS::AppSync::FunctionConfiguration',
+              Properties: {
+                Name: 'my-function',
+                ApiId: 'apiId',
+                DataSourceName: 'my-datasource',
+                FunctionVersion: '2018-05-29',
+                RequestMappingTemplate: '## original request template',
+                ResponseMappingTemplate: '## new response template',
+              },
+              Metadata: {
+                'aws:asset:path': 'new-path',
+              },
+            },
+          },
+        },
+      });
+
+      // WHEN
+      await expect(() => hotswapMockSdkProvider.tryHotswapDeployment(hotswapMode, cdkStackArtifact)).rejects.toThrow(
+        'ConcurrentModificationException',
+      );
+
+      // THEN
+      expect(mockAppSyncClient).toHaveReceivedCommandTimes(UpdateFunctionCommand, 7); // 1st attempt and then 6 retries before bailing
+      expect(mockAppSyncClient).toHaveReceivedCommandWith(UpdateFunctionCommand, {
+        apiId: 'apiId',
+        dataSourceName: 'my-datasource',
+        functionId: 'functionId',
+        functionVersion: '2018-05-29',
+        name: 'my-function',
+        requestMappingTemplate: '## original request template',
+        responseMappingTemplate: '## new response template',
+      });
+    },
+    320000,
+  );
+
   silentTest('calls the updateFunction() API with functionId when function is listed on second page', async () => {
     // GIVEN
     mockAppSyncClient