chore(api): trigger rds tagged release

.circleci/config.base.yml

@@ -112,6 +112,10 @@ jobs:
       - run:
           name: Build tests
           command: yarn build-tests
+      - run:
+          # setting versions in build step so that commit and git tags are cached and available for all steps in pipeline
+          name: Set versions
+          command: ./.circleci/publish-step-1-set-versions.sh
       - save_cache:
           key: amplify-cli-repo-{{ .Branch }}-{{ .Revision }}
           paths:
@@ -276,8 +280,7 @@ jobs:
             source .circleci/local_publish_helpers.sh
             startLocalRegistry "$(pwd)/.circleci/verdaccio.yaml"
             setNpmRegistryUrlToLocal
-            export LOCAL_PUBLISH_TO_LATEST=true
-            ./.circleci/publish.sh
+            ./.circleci/publish-step-2-verdaccio.sh
             unsetNpmRegistryUrl
       - run:
           name: Generate unified changelog
@@ -563,36 +566,6 @@ jobs:
           path: ~/repo/packages/amplify-migration-tests/amplify-migration-reports
     working_directory: ~/repo
 
-  amplify_migration_tests_v6:
-    parameters:
-      os:
-        type: executor
-        default: l_large
-    executor: << parameters.os >>
-    environment:
-      AMPLIFY_PATH: /home/circleci/.npm-global/lib/node_modules/@aws-amplify/cli/bin/amplify
-    steps:
-      - restore_cache:
-          key: amplify-cli-repo-{{ .Branch }}-{{ .Revision }}
-      - restore_cache:
-          key: amplify-cli-yarn-deps-{{ .Branch }}-{{ checksum "yarn.lock" }}
-      - restore_cache:
-          key: amplify-pkg-binaries-{{ .Branch }}-{{ .Revision }}
-      - run:
-          name: Run tests migrating from CLI v6.1.0
-          command: |
-            source .circleci/local_publish_helpers.sh
-            changeNpmGlobalPath
-            cd packages/amplify-migration-tests
-            retry yarn run migration_v6.1.0 --no-cache --maxWorkers=4 --forceExit $TEST_SUITE
-          no_output_timeout: 65m
-      - run: *scan_e2e_test_artifacts
-      - store_test_results:
-          path: ~/repo/packages/amplify-migration-tests/
-      - store_artifacts:
-          path: ~/repo/packages/amplify-migration-tests/amplify-migration-reports
-    working_directory: ~/repo
-
   amplify_migration_tests_v10:
     parameters:
       os:
@@ -850,7 +823,7 @@ jobs:
       - store_artifacts:
           path: ~/aws-amplify-cypress-api/cypress/screenshots
 
-  deploy:
+  publish_to_npm:
     <<: *linux-e2e-executor-large
     steps:
       - restore_cache:
@@ -860,20 +833,27 @@ jobs:
             - amplify-cli-ssh-deps-{{ .Branch }}
       - restore_cache:
           key: amplify-pkg-binaries-{{ .Branch }}-{{ .Revision }}
-      - run:
-          name: Upload Pkg Binary
-          command: |
-            source .circleci/local_publish_helpers.sh
-            uploadPkgCli
-            ./out/amplify-pkg-linux-x64 --version
       - run:
           name: Authenticate with npm
           command: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/.npmrc
       - run:
           name: Publish Amplify CLI
           command: |
-            bash ./.circleci/publish.sh
-      - run: *scan_e2e_test_artifacts
+            bash ./.circleci/publish-step-3-npm.sh
+  post_publish_push_to_git:
+    <<: *linux-e2e-executor-large
+    steps:
+      - restore_cache:
+          key: amplify-cli-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_cache:
+          keys:
+            - amplify-cli-ssh-deps-{{ .Branch }}
+      - restore_cache:
+          key: amplify-pkg-binaries-{{ .Branch }}-{{ .Revision }}
+      - run:
+          name: Push release commit and tags
+          command: |
+            bash ./.circleci/publish-step-4-push-to-git.sh
   github_prerelease:
     <<: *linux-e2e-executor-large
     steps:
@@ -934,6 +914,19 @@ jobs:
             commit=$(git rev-parse HEAD)
             version=$(cat .amplify-pkg-version)
             yarn ts-node scripts/github-release.ts $version $commit
+  npm_rollback_access:
+    <<: *linux-e2e-executor-large
+    steps:
+      - restore_cache:
+          key: amplify-cli-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_cache:
+          keys:
+            - amplify-cli-ssh-deps-{{ .Branch }}
+      - restore_cache:
+          key: amplify-pkg-binaries-{{ .Branch }}-{{ .Revision }}
+      - run:
+          name: Authenticate with npm
+          command: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/.npmrc
   cleanup_resources:
     <<: *linux-e2e-executor-large
     steps:
@@ -1285,11 +1278,8 @@ workflows:
                 - /tagged-release\/.*/
                 - /run-e2e\/.*/
 
-      - deploy:
+      - publish_to_npm:
           context:
-            - e2e-auth-credentials
-            - e2e-test-context
-            - amplify-s3-upload
             - npm-publish
           requires:
             - test
@@ -1314,10 +1304,31 @@ workflows:
                 - /run-e2e-with-rc\/.*/
                 - /tagged-release\/.*/
                 - /tagged-release-without-e2e-tests\/.*/
+      - npm_rollback_access:
+          context:
+            - npm-publish
+          requires:
+            - upload_pkg_binaries
+            - verify-versions-match
+          filters:
+            branches:
+              only:
+                - release
+      - post_publish_push_to_git:
+          requires:
+            - publish_to_npm
+          filters:
+            branches:
+              only:
+                - release
+                - /release_rc\/.*/
+                - /run-e2e-with-rc\/.*/
+                - /tagged-release\/.*/
+                - /tagged-release-without-e2e-tests\/.*/
       - github_release:
           context: github-publish
           requires:
-            - deploy
+            - post_publish_push_to_git
           filters:
             branches:
               only:

.circleci/local_publish_helpers.sh

@@ -5,11 +5,22 @@ default_verdaccio_package=verdaccio@5.1.2
 
 function startLocalRegistry {
     # Start local registry
-    tmp_registry_log=$(mktemp)
+    tmp_registry_log="$(mktemp)"
     echo "Registry output file: $tmp_registry_log"
     (cd && nohup npx ${VERDACCIO_PACKAGE:-$default_verdaccio_package} -c $1 &>$tmp_registry_log &)
     # Wait for Verdaccio to boot
-    grep -q 'http address' <(tail -f $tmp_registry_log)
+    attempts=0
+    until grep -q 'http address' $tmp_registry_log
+    do
+      attempts=$((attempts+1))
+      echo "Waiting for Verdaccio, attempt $attempts"
+      sleep 1
+
+      if (( attempts > 60 )); then
+        echo "Verdaccio didn't start";
+        exit 1
+      fi
+    done
 }
 
 function uploadPkgCli {
@@ -152,7 +163,7 @@ function unsetSudoNpmRegistryUrl {
 }
 
 function changeNpmGlobalPath {
-    mkdir -p ~/.npm-global
+    mkdir -p ~/.npm-global/{bin,lib}
     npm config set prefix '~/.npm-global'
     export PATH=~/.npm-global/bin:$PATH
 }
@@ -266,7 +277,6 @@ function setAwsAccountCredentials {
         useChildAccountCredentials
     fi
 }
-
 function runE2eTest {
     FAILED_TEST_REGEX_FILE="./amplify-e2e-reports/amplify-e2e-failed-test.txt"
 
@@ -279,6 +289,36 @@ function runE2eTest {
     fi
 }
 
+function runE2eTestCb {
+    _setupCoverage
+    FAILED_TEST_REGEX_FILE="./amplify-e2e-reports/amplify-e2e-failed-test.txt"
+
+    if [ -f  $FAILED_TEST_REGEX_FILE ]; then
+        # read the content of failed tests
+        failedTests=$(<$FAILED_TEST_REGEX_FILE)
+        NODE_V8_COVERAGE=$E2E_TEST_COVERAGE_DIR yarn e2e --forceExit --no-cache --maxWorkers=4 $TEST_SUITE -t "$failedTests"
+    else
+        NODE_V8_COVERAGE=$E2E_TEST_COVERAGE_DIR yarn e2e --forceExit --no-cache --maxWorkers=4 $TEST_SUITE
+    fi
+}
+
+function _setupCoverage {
+    _teardownCoverage
+    echo "Setup Coverage ($E2E_TEST_COVERAGE_DIR)"
+    if [ ! -d $E2E_TEST_COVERAGE_DIR ]
+    then
+        mkdir -p $E2E_TEST_COVERAGE_DIR
+    fi
+}
+
+function _teardownCoverage {
+    if [ -d $E2E_TEST_COVERAGE_DIR ]
+    then
+        echo "Teardown Coverage ($E2E_TEST_COVERAGE_DIR)"
+        rm -r $E2E_TEST_COVERAGE_DIR
+    fi
+}
+
 function checkPackageVersionsInLocalNpmRegistry {
     cli_internal_version=$(npm view @aws-amplify/cli-internal version)
     cli_version=$(npm view @aws-amplify/cli version)

.circleci/publish.sh → .circleci/publish-codebuild.sh

@@ -1,23 +1,46 @@
 #!/bin/bash -e
+export BRANCH_NAME="$(git symbolic-ref HEAD --short 2>/dev/null)"
+if [ "$BRANCH_NAME" = "" ] ; then
+  BRANCH_NAME="$(git rev-parse HEAD | xargs git name-rev | cut -d' ' -f2 | sed 's/remotes\/origin\///g')";
+fi
+git checkout $BRANCH_NAME
+echo "fetching tags"
+git fetch --tags https://github.com/aws-amplify/amplify-cli
 
 # lerna has a bug (https://github.com/lerna/lerna/issues/1066) where failed publishes do not set the exit code properly
 # this causes the script to keep running even after failed publishes
 # this function forces failed publishes to exit on failure
 function lernaPublishExitOnFailure {
   # exit on failure
   set -e
-  # run lerna publish with the args that were passed to this function
-  # duplicate stdout to a temp file
-  # grep the temp file for the lerna err token and return exit 1 if found (-v option inverts grep exit code)
-
   if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
     # registy URL update changes .yarnrc.yml file
     git update-index --assume-unchanged .yarnrc.yml
   fi
 
+  # run lerna publish with the args that were passed to this function
+  # duplicate stdout to a temp file
+  # grep the temp file for the lerna err token and return exit 1 if found (-v option inverts grep exit code)
   npx lerna publish "$@" | tee /tmp/publish-results && grep -qvz "lerna ERR!" < /tmp/publish-results
 }
 
+# verifies that binaries are uploaded and available before publishing to NPM
+function verifyPkgIsAvailable {
+  # exit on failure
+  set -e
+
+  # read version of @aws-amplify/cli
+  desiredPkgVersion=$(npx lerna list --scope @aws-amplify/cli --json | jq -r '.[0].version')
+
+  # check binaries
+  # send HEAD requests to check for binary presence
+  # curl --fail exits with non-zero code and makes this script fail
+  curl -I --fail  https://package.cli.amplify.aws/$desiredPkgVersion/amplify-pkg-linux-x64.tgz
+  curl -I --fail  https://package.cli.amplify.aws/$desiredPkgVersion/amplify-pkg-linux-arm64.tgz
+  curl -I --fail  https://package.cli.amplify.aws/$desiredPkgVersion/amplify-pkg-macos-x64.tgz
+  curl -I --fail  https://package.cli.amplify.aws/$desiredPkgVersion/amplify-pkg-win-x64.tgz
+}
+
 if [ -z "$GITHUB_EMAIL" ]; then
   if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
     git config --global user.email not@used.com
@@ -40,13 +63,13 @@ else
   git config --global user.name $GITHUB_USER
 fi
 
-if [[ "$CIRCLE_BRANCH" =~ ^tagged-release ]]; then
-  if [[ "$CIRCLE_BRANCH" =~ ^tagged-release-without-e2e-tests\/.* ]]; then
+if [[ "$BRANCH_NAME" =~ ^tagged-release ]]; then
+  if [[ "$BRANCH_NAME" =~ ^tagged-release-without-e2e-tests\/.* ]]; then
     # Remove tagged-release-without-e2e-tests/
-    export NPM_TAG="${CIRCLE_BRANCH/tagged-release-without-e2e-tests\//}"
-  elif [[ "$CIRCLE_BRANCH" =~ ^tagged-release\/.* ]]; then
+    export NPM_TAG="${BRANCH_NAME/tagged-release-without-e2e-tests\//}"
+  elif [[ "$BRANCH_NAME" =~ ^tagged-release\/.* ]]; then
     # Remove tagged-release/
-    export NPM_TAG="${CIRCLE_BRANCH/tagged-release\//}"
+    export NPM_TAG="${BRANCH_NAME/tagged-release\//}"
   fi
   if [ -z "$NPM_TAG" ]; then
     echo "Tag name is missing. Name your branch with either tagged-release/<tag-name> or tagged-release-without-e2e-tests/<tag-name>"
@@ -62,10 +85,15 @@ if [[ "$CIRCLE_BRANCH" =~ ^tagged-release ]]; then
   fi
 
 # @latest release
-elif [[ "$CIRCLE_BRANCH" == "release" ]]; then
+elif [[ "$BRANCH_NAME" == "release" ]]; then
   # create release commit and release tags
   npx lerna version --exact --conventional-commits --conventional-graduate --yes --no-push --include-merged-tags --message "chore(release): Publish latest [ci skip]"
 
+  if [[ "$LOCAL_PUBLISH_TO_LATEST" != "true" ]]; then
+    # verify that binary has been uploaded
+    verifyPkgIsAvailable
+  fi
+
   # publish versions that were just computed
   lernaPublishExitOnFailure from-git --yes --no-push
 
@@ -76,7 +104,7 @@ elif [[ "$CIRCLE_BRANCH" == "release" ]]; then
   fi
 
   # push release commit
-  git push origin "$CIRCLE_BRANCH"
+  git push origin "$BRANCH_NAME"
 
   # push release tags
   git tag --points-at HEAD | xargs git push origin
@@ -94,7 +122,7 @@ elif [[ "$CIRCLE_BRANCH" == "release" ]]; then
   git push origin hotfix
 
 # release candidate or local publish for testing / building binary
-elif [[ "$CIRCLE_BRANCH" =~ ^run-e2e-with-rc\/.* ]] || [[ "$CIRCLE_BRANCH" =~ ^release_rc\/.* ]] || [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
+elif [[ "$BRANCH_NAME" =~ ^run-e2e-with-rc\/.* ]] || [[ "$BRANCH_NAME" =~ ^release_rc\/.* ]] || [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
 
   # force @aws-amplify/cli-internal to be versioned in case this pipeline run does not have any commits that modify the CLI packages
   if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
@@ -103,6 +131,7 @@ elif [[ "$CIRCLE_BRANCH" =~ ^run-e2e-with-rc\/.* ]] || [[ "$CIRCLE_BRANCH" =~ ^r
   # create release commit and release tags
   npx lerna version --preid=rc.$(git rev-parse --short HEAD) --exact --conventional-prerelease --conventional-commits --yes --no-push --include-merged-tags --message "chore(release): Publish rc [ci skip]" $(echo $force_publish_local_args) --no-commit-hooks
 
+
   # if publishing locally to verdaccio
   if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
     # publish to verdaccio with no dist tag (default to latest)
@@ -112,14 +141,17 @@ elif [[ "$CIRCLE_BRANCH" =~ ^run-e2e-with-rc\/.* ]] || [[ "$CIRCLE_BRANCH" =~ ^r
     exit 0
   fi
 
+  # verify that binary has been uploaded
+  verifyPkgIsAvailable
+
   # publish versions that were just computed
   lernaPublishExitOnFailure from-git --yes --no-push --dist-tag rc
 
   # push release commit
-  git push origin "$CIRCLE_BRANCH"
+  git push origin "$BRANCH_NAME"
 
   # push release tags
   git tag --points-at HEAD | xargs git push origin
 else
-  echo "branch name" "$CIRCLE_BRANCH" "did not match any branch publish rules. Skipping publish"
+  echo "branch name" "$BRANCH_NAME" "did not match any branch publish rules. Skipping publish"
 fi