Build changes - Add GitShortSha to config, Add md5, json file creation

build-infrastructure/release-pipeline-stack.yml

@@ -1043,23 +1043,17 @@ Resources:
               Configuration:
                 ProjectName: !Ref SigningCodeBuildProject
                 PrimarySource: Buildspecs
-                # This project takes the following names as env vars
-                # - Commit sha
-                # - AMD tar
-                # - AMD rpm
-                # - ECS Anywhere install script (From AMD Project)
-                # - Ubuntu AMD deb
-                # - Ubuntu ARM deb
-                # - ARM tar
-                # - ARM rpm
-                EnvironmentVariables: '[{"name":"GIT_COMMIT_SHA","value":"#{SourceVariables.CommitId}","type":"PLAINTEXT"},{"name":"ECS_AGENT_AMD_TAR","value":"#{AmdBuildVariables.ECS_AGENT_TAR}","type":"PLAINTEXT"},{"name":"ECS_AGENT_AMD_RPM","value":"#{AmdBuildVariables.ECS_AGENT_RPM}","type":"PLAINTEXT"},{"name":"ECS_AGENT_UBUNTU_AMD_DEB","value":"#{UbuntuAmdBuildVariables.ECS_AGENT_DEB}", "type":"PLAINTEXT"},{"name":"ECS_AGENT_UBUNTU_ARM_DEB","value":"#{UbuntuArmBuildVariables.ECS_AGENT_DEB}", "type":"PLAINTEXT"},{"name":"ECS_AGENT_ARM_TAR","value":"#{ArmBuildVariables.ECS_AGENT_TAR}","type":"PLAINTEXT"},{"name":"ECS_AGENT_ARM_RPM","value":"#{ArmBuildVariables.ECS_AGENT_RPM}","type":"PLAINTEXT"}]'
+                # Agent version and git short sha env variables to generate file names for signing
+                EnvironmentVariables: '[{"name":"AGENT_VERSION","value":"#{AmdBuildVariables.AGENT_VERSION}","type":"PLAINTEXT"},{"name":"GIT_COMMIT_SHORT_SHA","value":"#{AmdBuildVariables.GIT_COMMIT_SHORT_SHA}","type":"PLAINTEXT"},{"name":"INIT_VERSION","value":"#{AmdBuildVariables.INIT_VERSION}","type":"PLAINTEXT"}]'
               OutputArtifacts:
                 - Name: SignedArtifact
               RunOrder: 1
               Namespace: SigningVariables
         - Name: ReleaseConfig
           Actions:
             - Name: MakeJSON
+              # SourceArtifact as input to access the github repository
+              # To run git short sha command in buildspec
               InputArtifacts:
                 - Name: Buildspecs
                 - Name: SignedArtifact
@@ -1071,7 +1065,7 @@ Resources:
               Configuration:
                 ProjectName: !Ref MakeJSONCodeBuildProject
                 PrimarySource: Buildspecs
-                EnvironmentVariables: '[{"name":"GIT_COMMIT_SHA","value":"#{SourceVariables.CommitId}","type":"PLAINTEXT"},{"name":"AGENT_VERSION","value":"#{AmdBuildVariables.AGENT_VERSION}","type":"PLAINTEXT"}]'
+                EnvironmentVariables: '[{"name":"GIT_COMMIT_SHA","value":"#{SourceVariables.CommitId}","type":"PLAINTEXT"},{"name":"AGENT_VERSION","value":"#{AmdBuildVariables.AGENT_VERSION}","type":"PLAINTEXT"},{"name":"GIT_COMMIT_SHORT_SHA","value":"#{AmdBuildVariables.GIT_COMMIT_SHORT_SHA}","type":"PLAINTEXT"}]'
               OutputArtifacts:
                 - Name: JSONArtifact
               RunOrder: 1
@@ -1083,6 +1077,8 @@ Resources:
                 - Name: Buildspecs
                 - Name: SignedArtifact
                 - Name: JSONArtifact
+                - Name: AmdBuildArtifact
+                - Name: ArmBuildArtifact
               ActionTypeId:
                 Category: Build
                 Owner: AWS

buildspecs/copy.yml

@@ -20,3 +20,19 @@ phases:
           echo "copying $filename to destination $RESULTS_BUCKET_URI/$GIT_COMMIT_SHA"
           aws s3 cp $filename "$RESULTS_BUCKET_URI/$GIT_COMMIT_SHA/$filename"
         done
+
+      # copy amd md5 and json files
+      - cd $CODEBUILD_SRC_DIR_AmdBuildArtifact
+      - |
+        for filename in  *.md5 *.json; do
+          echo "copying $filename to destination $RESULTS_BUCKET_URI/$GIT_COMMIT_SHA"
+          aws s3 cp $filename "$RESULTS_BUCKET_URI/$GIT_COMMIT_SHA/$filename"
+        done
+
+      # copy amd md5 and json files
+      - cd $CODEBUILD_SRC_DIR_ArmBuildArtifact
+      - |
+        for filename in  *.md5 *.json; do
+          echo "copying $filename to destination $RESULTS_BUCKET_URI/$GIT_COMMIT_SHA"
+          aws s3 cp $filename "$RESULTS_BUCKET_URI/$GIT_COMMIT_SHA/$filename"
+        done

buildspecs/merge-build-ubuntu.yml

@@ -26,7 +26,7 @@ phases:
     commands:
       - echo "Building agent deb"
       - AGENT_VERSION=$(cat VERSION)
-      - ECS_AGENT_DEB="amazon-ecs-init_${AGENT_VERSION}-1_${architecture}.deb"
+      - ECS_AGENT_DEB="amazon-ecs-init-${AGENT_VERSION}-1.${architecture}.deb"
       - ECS_AGENT_DEB_TAR="amazon-ecs-init_${AGENT_VERSION}-1.debian.tar.xz"
       - echo $(pwd)
 
@@ -47,6 +47,7 @@ phases:
       - which go
       - go version
       - make generic-deb-integrated
+      - mv amazon-ecs-init_${AGENT_VERSION}-1_${architecture}.deb $ECS_AGENT_DEB
 
 artifacts:
   files:

buildspecs/merge-build.yml

@@ -4,9 +4,10 @@ env:
   git-credential-helper: yes
   exported-variables:
     - CODEBUILD_BUILD_ID
-    - ECS_AGENT_TAR
     - ECS_AGENT_RPM
     - AGENT_VERSION
+    - INIT_VERSION
+    - GIT_COMMIT_SHORT_SHA
 
 phases:
   install:
@@ -47,11 +48,18 @@ phases:
       # Read init version from changelog, using this as the source because of possible scenario of '-2', '-3'.. init suffix releases
       - INIT_VERSION=$(head -n 1 scripts/changelog/CHANGELOG_MASTER)
       - INIT_VERSION=$(echo $INIT_VERSION | tr -d '[:space:]')
+
+      # Git short sha - used to name artifacts and make release json
+      # This variable is exported
+      - GIT_COMMIT_SHORT_SHA=$(git rev-parse --short HEAD)
+
+      # Different names of under which tar is released
       - ECS_AGENT_TAR="ecs-agent-v${AGENT_VERSION}.tar"
+      - ECS_AGENT_LATEST_TAR="ecs-agent-latest.tar"
+      - ECS_AGENT_GITSHORTSHA_TAR="ecs-agent-${GIT_COMMIT_SHORT_SHA}.tar"
+
+      # RPM
       - ECS_AGENT_RPM="amazon-ecs-init-${INIT_VERSION}.x86_64.rpm"
-      - echo $(pwd)
-      - RELEASE_DATE=$(git show -s --format=%cd --date=format:'%Y%m%d')
-      - echo $RELEASE_DATE
 
       # Directory/GOPATH restructuring needed for CodePipeline
       - cd ../..
@@ -67,16 +75,58 @@ phases:
       - make generic-rpm-integrated
       - ls
 
-      # Rename artifacts for architecture
+      # Rename artifacts for arm architecture
       - |
         if [[ $architecture == "arm64" ]] ; then
           mv $ECS_AGENT_TAR "ecs-agent-arm64-v${AGENT_VERSION}.tar"
           ECS_AGENT_RPM="amazon-ecs-init-${INIT_VERSION}.aarch64.rpm"
           ECS_AGENT_TAR="ecs-agent-arm64-v${AGENT_VERSION}.tar"
+          ECS_AGENT_LATEST_TAR="ecs-agent-arm64-latest.tar"
+          ECS_AGENT_GITSHORTSHA_TAR="ecs-agent-arm64-${GIT_COMMIT_SHORT_SHA}.tar"
         fi
 
+      # Make copies of agent under different names; Same tar - different names
+      - cp $ECS_AGENT_TAR $ECS_AGENT_LATEST_TAR
+      - cp $ECS_AGENT_TAR $ECS_AGENT_GITSHORTSHA_TAR
+
+      # md5 file names
+      - ECS_AGENT_TAR_MD5="${ECS_AGENT_TAR}.md5"
+      - ECS_AGENT_LATEST_TAR_MD5="${ECS_AGENT_LATEST_TAR}.md5"
+      - ECS_AGENT_GITSHORTSHA_TAR_MD5="${ECS_AGENT_GITSHORTSHA_TAR}.md5"
+
+      # Create md5 file names
+      - md5sum $ECS_AGENT_TAR | awk '{print $1}' > $ECS_AGENT_TAR_MD5
+      - md5sum $ECS_AGENT_LATEST_TAR | awk '{print $1}' > $ECS_AGENT_LATEST_TAR_MD5
+      - md5sum $ECS_AGENT_GITSHORTSHA_TAR | awk '{print $1}' > $ECS_AGENT_GITSHORTSHA_TAR_MD5
+
+      # json file names 
+      - ECS_AGENT_TAR_JSON="${ECS_AGENT_TAR}.json"
+      - ECS_AGENT_LATEST_TAR_JSON="${ECS_AGENT_LATEST_TAR}.json"
+      - ECS_AGENT_GITSHORTSHA_TAR_JSON="${ECS_AGENT_GITSHORTSHA_TAR}.json"
+
+      # Create jsons
+      - echo "{\"agentVersion\":\"v${AGENT_VERSION}\"}" | tee $ECS_AGENT_TAR_JSON $ECS_AGENT_LATEST_TAR_JSON $ECS_AGENT_GITSHORTSHA_TAR_JSON
+
+
 artifacts:
   files:
+    # tars
     - $ECS_AGENT_TAR
+    - $ECS_AGENT_LATEST_TAR
+    - $ECS_AGENT_GITSHORTSHA_TAR
+
+    # md5
+    - $ECS_AGENT_TAR_MD5
+    - $ECS_AGENT_LATEST_TAR_MD5
+    - $ECS_AGENT_GITSHORTSHA_TAR_MD5
+
+    # json
+    - $ECS_AGENT_TAR_JSON
+    - $ECS_AGENT_LATEST_TAR_JSON
+    - $ECS_AGENT_GITSHORTSHA_TAR_JSON
+
+    # rpm
     - $ECS_AGENT_RPM
+
+    # ECS Anywhere install script
     - 'scripts/ecs-anywhere-install.sh'

buildspecs/release-config.yml

@@ -34,6 +34,7 @@ phases:
           "releaseDate" : "$RELEASE_DATE",
           "agentStagingConfig": {
             "releaseGitSha": "$GIT_COMMIT_SHA",
+            "releaseGitShortSha": "$GIT_COMMIT_SHORT_SHA",
             "gitFarmRepoName": "MadisonContainerAgentExternal",
             "gitHubRepoName": "aws/amazon-ecs-agent",
             "gitFarmStageBranch": "v${AGENT_VERSION}-stage",

buildspecs/signing.yml

@@ -41,6 +41,21 @@ phases:
   build:
     on-failure: ABORT
     commands:
+      # Generate file names for signing from agent version and git short sha
+      - ECS_AGENT_AMD_TAR="ecs-agent-v${AGENT_VERSION}.tar"
+      - ECS_AGENT_AMD_LATEST_TAR="ecs-agent-latest.tar"
+      - ECS_AGENT_AMD_GITSHORTSHA_TAR="ecs-agent-${GIT_COMMIT_SHORT_SHA}.tar"
+      - ECS_AGENT_AMD_RPM="amazon-ecs-init-${INIT_VERSION}.x86_64.rpm"
+      - ECS_AGENT_UBUNTU_AMD_DEB="amazon-ecs-init-${INIT_VERSION}.amd64.deb"
+
+      - ECS_AGENT_ARM_TAR="ecs-agent-arm64-v${AGENT_VERSION}.tar"
+      - ECS_AGENT_ARM_LATEST_TAR="ecs-agent-arm64-latest.tar"
+      - ECS_AGENT_ARM_GITSHORTSHA_TAR="ecs-agent-arm64-${GIT_COMMIT_SHORT_SHA}.tar"
+      - ECS_AGENT_ARM_RPM="amazon-ecs-init-${INIT_VERSION}.aarch64.rpm"
+      - ECS_AGENT_UBUNTU_ARM_DEB="amazon-ecs-init-${INIT_VERSION}.arm64.deb"
+
+      - ECS_ANYWHERE_SCRIPT="ecs-anywhere-install-${INIT_VERSION}.sh"
+      - ECS_ANYWHERE_LATEST_SCRIPT="ecs-anywhere-install-latest.sh"
       # Get the private key from secrets manager, jq parse it, turn it into raw output, pipe to file
       - aws secretsmanager get-secret-value --secret-id $PRIVATE_KEY_ARN | jq -r '.SecretString' > private.gpg
       # import the key into the keychain, the private key comes with the public key built in
@@ -50,14 +65,24 @@ phases:
       # Sign the amd tar and rpm (this is a secondary source so we have to do some copying)
       - cp "$CODEBUILD_SRC_DIR_AmdBuildArtifact/$ECS_AGENT_AMD_TAR" $ECS_AGENT_AMD_TAR
       - source /tmp/functions.sh && sign_file $ECS_AGENT_AMD_TAR
+      - cp "$CODEBUILD_SRC_DIR_AmdBuildArtifact/$ECS_AGENT_AMD_LATEST_TAR" $ECS_AGENT_AMD_LATEST_TAR
+      - source /tmp/functions.sh && sign_file $ECS_AGENT_AMD_LATEST_TAR
+      - cp "$CODEBUILD_SRC_DIR_AmdBuildArtifact/$ECS_AGENT_AMD_GITSHORTSHA_TAR" $ECS_AGENT_AMD_GITSHORTSHA_TAR
+      - source /tmp/functions.sh && sign_file $ECS_AGENT_AMD_GITSHORTSHA_TAR
       - cp "$CODEBUILD_SRC_DIR_AmdBuildArtifact/$ECS_AGENT_AMD_RPM" $ECS_AGENT_AMD_RPM
       - source /tmp/functions.sh && sign_file $ECS_AGENT_AMD_RPM
       # Sign ECS Anywhere Script
-      - cp "$CODEBUILD_SRC_DIR_AmdBuildArtifact/scripts/ecs-anywhere-install.sh" ecs-anywhere-install.sh
-      - source /tmp/functions.sh && sign_file ecs-anywhere-install.sh
+      - cp "$CODEBUILD_SRC_DIR_AmdBuildArtifact/scripts/ecs-anywhere-install.sh" $ECS_ANYWHERE_SCRIPT
+      - source /tmp/functions.sh && sign_file $ECS_ANYWHERE_SCRIPT
+      - cp $ECS_ANYWHERE_SCRIPT $ECS_ANYWHERE_LATEST_SCRIPT
+      - source /tmp/functions.sh && sign_file $ECS_ANYWHERE_LATEST_SCRIPT
       # Sign the arm tar and rpm (this is a secondary source so we have to do some copying)
       - cp "$CODEBUILD_SRC_DIR_ArmBuildArtifact/$ECS_AGENT_ARM_TAR" $ECS_AGENT_ARM_TAR
       - source /tmp/functions.sh && sign_file $ECS_AGENT_ARM_TAR
+      - cp "$CODEBUILD_SRC_DIR_ArmBuildArtifact/$ECS_AGENT_ARM_LATEST_TAR" $ECS_AGENT_ARM_LATEST_TAR
+      - source /tmp/functions.sh && sign_file $ECS_AGENT_ARM_LATEST_TAR
+      - cp "$CODEBUILD_SRC_DIR_ArmBuildArtifact/$ECS_AGENT_ARM_GITSHORTSHA_TAR" $ECS_AGENT_ARM_GITSHORTSHA_TAR
+      - source /tmp/functions.sh && sign_file $ECS_AGENT_ARM_GITSHORTSHA_TAR
       - cp "$CODEBUILD_SRC_DIR_ArmBuildArtifact/$ECS_AGENT_ARM_RPM" $ECS_AGENT_ARM_RPM
       - source /tmp/functions.sh && sign_file $ECS_AGENT_ARM_RPM
       # Sign the amd deb (this is a secondary source so we have to do some copying)
@@ -75,15 +100,25 @@ artifacts:
   files:
     - $ECS_AGENT_AMD_TAR
     - '$ECS_AGENT_AMD_TAR.asc'
+    - $ECS_AGENT_AMD_LATEST_TAR
+    - '$ECS_AGENT_AMD_LATEST_TAR.asc'
+    - $ECS_AGENT_AMD_GITSHORTSHA_TAR
+    - '$ECS_AGENT_AMD_GITSHORTSHA_TAR.asc'
     - $ECS_AGENT_AMD_RPM
     - '$ECS_AGENT_AMD_RPM.asc'
     - $ECS_AGENT_ARM_TAR
     - '$ECS_AGENT_ARM_TAR.asc'
+    - $ECS_AGENT_ARM_LATEST_TAR
+    - '$ECS_AGENT_ARM_LATEST_TAR.asc'
+    - $ECS_AGENT_ARM_GITSHORTSHA_TAR
+    - '$ECS_AGENT_ARM_GITSHORTSHA_TAR.asc'
     - $ECS_AGENT_ARM_RPM
     - '$ECS_AGENT_ARM_RPM.asc'
     - $ECS_AGENT_UBUNTU_AMD_DEB
     - '$ECS_AGENT_UBUNTU_AMD_DEB.asc'
     - $ECS_AGENT_UBUNTU_ARM_DEB
     - '$ECS_AGENT_UBUNTU_ARM_DEB.asc'
-    - 'ecs-anywhere-install.sh'
-    - 'ecs-anywhere-install.sh.asc'
+    - $ECS_ANYWHERE_SCRIPT
+    - '$ECS_ANYWHERE_SCRIPT.asc'
+    - $ECS_ANYWHERE_LATEST_SCRIPT
+    - '$ECS_ANYWHERE_LATEST_SCRIPT.asc'