[INF][OPS] Use stripped assembly for pipeline CFN actions

Works around the 256MB input artifact size limit for CFN deploy actions in CodePipeline, which can be exceeded due to asset files.

Related to aws/aws-cdk#9917.

buildspec.yml

@@ -15,6 +15,7 @@ phases:
     runtime-versions:
       python: 3.8
     commands:
+      - "apt install -y jq moreutils"
       # Install NPM to 'global' local repo
       - "npm install -g aws-cdk@${CDK_VERSION}"
       # Download Graal only if not already installed (cached by version/GRAAL_RELEASE)
@@ -50,17 +51,27 @@ phases:
         )
       # Synthesize application/service CDK stacks for application deployment
       - "mkdir -p gen/cloud-assembly"
-      - "(cd deployment/application-infrastructure/ && cdk synth -o ../../gen/cloud-assembly)"
+      - "(cd deployment/application-infrastructure/ && cdk synth -o ../../gen/cloud_assembly)"
       - "echo \"Generated $(cat gen/cloud-assembly/*.template.json | wc -l) template lines\""
+      # Create the no-assets cloned assembly for CFN pipeline actions
+      - "buildtools/scripts/clone_assembly.sh"
   post_build:
     commands:
       - "echo \"Build completed at $(date)\""
 artifacts:
-  name: "cloud_assembly"
-  base-directory: "gen/cloud-assembly"
-  discard-paths: no
   files:
-    - "**/*"
+    - "**.*" # Useless but required field
+  secondary-artifacts:
+    cloud_assembly:
+      base-directory: "gen/cloud_assembly"
+      discard-paths: no
+      files:
+        - "**/*"
+    cloud_assembly_no_assets:
+      base-directory: "gen/cloud_assembly_no_assets"
+      discard-paths: no
+      files:
+        - "**/*"
 cache:
   paths:
     - '/root/.m2/**/*'

buildtools/scripts/clone_assembly.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Slightly hacky script to work around the 256MB input artifact size limit for CFN pipeline actions.
+# Ref: https://docs.aws.amazon.com/codepipeline/latest/userguide/limits.html
+#
+# Since assets are what makes the artifact so big, we use a separate, cloned artifact with assets removed.
+# Relevant GitHub issue: https://github.com/aws/aws-cdk/issues/9917
+
+# Make a copy of the cloud assembly but exclude any asset files
+rsync -av gen/cloud_assembly/ gen/cloud_assembly_no_assets/ --exclude '/asset.*'
+
+# Prep a JQ expression to update all pipeline CFN actions to use the no-assets artifact
+JQ_PIPELINE_ACTIONS_PATH='.Resources.DeploymentPipelineD53681EF.Properties.Stages[].Actions[]'
+JQ_ACTION_IS_CFN_TYPE='.ActionTypeId.Provider == "CloudFormation"'
+JQ_OVERWRITE_ARTIFACT_EXPR='.InputArtifacts = [{Name:"cloud_assembly_no_assets"}]'
+JQ_EXPR="$JQ_PIPELINE_ACTIONS_PATH |= if ($JQ_ACTION_IS_CFN_TYPE) then ($JQ_OVERWRITE_ARTIFACT_EXPR) else . end"
+
+# Edit the pipeline stack template file in-place
+PIPELINE_STACK_FILE='gen/cloud_assembly/DeploymentPipelineStack.template.json'
+jq "$JQ_EXPR" $PIPELINE_STACK_FILE | sponge $PIPELINE_STACK_FILE

common/shared-configuration/src/main/java/com/admiralbot/sharedconfig/DeployConfig.java

@@ -15,4 +15,7 @@ public class DeployConfig {
 
     public static final String SERVICE_LOGS_PREFIX = "serverbot2/svc";
 
+    public static final String CLOUD_ASSEMBLY_ARTIFACT_NAME = "cloud_assembly";
+    public static final String CLOUD_ASSEMBLY_NO_ASSETS_ARTIFACT_NAME = "cloud_assembly_no_assets";
+
 }

deployment/application-infrastructure/src/main/java/com/admiralbot/infra/deploy/PipelineStack.java

@@ -42,10 +42,11 @@ public PipelineStack(Construct parent, String id, StackProps stackProps, String
 
         // Inputs:
         Artifact sourceArtifact = Artifact.artifact("github_source");
-        // Artifact manifestArtifact = Artifact.artifact("environment_manifest");
 
         // Outputs:
-        Artifact assemblyArtifact = Artifact.artifact("cloud_assembly");
+        Artifact assemblyArtifact = Artifact.artifact(DeployConfig.CLOUD_ASSEMBLY_ARTIFACT_NAME);
+        // Though the Java CDK code doesn't appear to use this, it's overwritten into the output CFN so we need to build it.
+        Artifact assemblyArtifactNoAssets = Artifact.artifact(DeployConfig.CLOUD_ASSEMBLY_NO_ASSETS_ARTIFACT_NAME);
 
         String connectionArn = Joiner.colon("arn", "aws", "codestar-connections",
                 getRegion(), getAccount(), "connection/" + connectionUuid);
@@ -90,7 +91,7 @@ public PipelineStack(Construct parent, String id, StackProps stackProps, String
                 .project(codeBuildProject)
                 .actionName("BuildAndSynthProject")
                 .input(sourceArtifact)
-                .outputs(List.of(assemblyArtifact))
+                .outputs(List.of(assemblyArtifact, assemblyArtifactNoAssets))
                 .build();
 
         pipeline = CdkPipeline.Builder.create(this, "DeploymentPipeline")