ci: build graph skeleton (#12278)

* chore: cache sample

* chore: update sample cache

* chore: test

* chore: test artifacts

* chore: rename file

* ci: testing s3 bucket

* chore: fix build issue

* ci: implement caching solution

* ci: test

* ci: fix config

* ci: try another way to run buildlinux

* ci: fix path

* ci: fix source

* ci: fix sh file header

* ci: switch source to dot

* ci: remove use of local

* ci: update s3 pathing

* ci: remove function

* ci: try again

* ci: specify bash

* ci: bash test

* ci: remove parenthesis

* ci: updated the cache to load

* ci: fix

* ci: mute cache

* ci: fix

* ci: update cache

* ci: cleanup

* ci: add cache file method, graph updates

* ci: graph updates

* ci: rune2e sample

* ci: yml fix

* ci: fix

* ci: fix

* ci: fix

* ci: add validate

* ci: fix

* ci: fix file names

* ci: fix publish

* ci: test

* ci: fix pwd

* ci: fix

* ci: fix

---------

Co-authored-by: Doesnt Matter <not@used.com>

.circleci/publish-codebuild.sh

@@ -0,0 +1,119 @@
+#!/bin/bash -e
+
+# lerna has a bug (https://github.com/lerna/lerna/issues/1066) where failed publishes do not set the exit code properly
+# this causes the script to keep running even after failed publishes
+# this function forces failed publishes to exit on failure
+function lernaPublishExitOnFailure {
+  # exit on failure
+  set -e
+  # run lerna publish with the args that were passed to this function
+  # duplicate stdout to a temp file
+  # grep the temp file for the lerna err token and return exit 1 if found (-v option inverts grep exit code)
+  npx lerna publish "$@" | tee /tmp/publish-results && grep -qvz "lerna ERR!" < /tmp/publish-results
+}
+
+if [ -z "$GITHUB_EMAIL" ]; then
+  if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
+    git config --global user.email not@used.com
+  else
+    echo "GITHUB_EMAIL email is missing"
+    exit 1
+  fi
+else
+  git config --global user.email $GITHUB_EMAIL
+fi
+
+if [ -z "$GITHUB_USER" ]; then
+  if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
+    git config --global user.name "Doesnt Matter"
+  else
+    echo "GITHUB_USER email is missing"
+    exit 1
+  fi
+else
+  git config --global user.name $GITHUB_USER
+fi
+
+if [[ "$CIRCLE_BRANCH" =~ ^tagged-release ]]; then
+  if [[ "$CIRCLE_BRANCH" =~ ^tagged-release-without-e2e-tests\/.* ]]; then
+    # Remove tagged-release-without-e2e-tests/
+    export NPM_TAG="${CIRCLE_BRANCH/tagged-release-without-e2e-tests\//}"
+  elif [[ "$CIRCLE_BRANCH" =~ ^tagged-release\/.* ]]; then
+    # Remove tagged-release/
+    export NPM_TAG="${CIRCLE_BRANCH/tagged-release\//}"
+  fi
+  if [ -z "$NPM_TAG" ]; then
+    echo "Tag name is missing. Name your branch with either tagged-release/<tag-name> or tagged-release-without-e2e-tests/<tag-name>"
+    exit 1
+  fi
+
+  if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
+    echo "Publishing to local registry under latest tag"
+    lernaPublishExitOnFailure --exact --preid=$NPM_TAG --conventional-commits --conventional-prerelease --no-push --yes --include-merged-tags
+  else
+    echo "Publishing to NPM under $NPM_TAG tag"
+    lernaPublishExitOnFailure --exact --dist-tag=$NPM_TAG --preid=$NPM_TAG --conventional-commits --conventional-prerelease --message "chore(release): Publish tagged release $NPM_TAG [ci skip]" --yes --include-merged-tags
+  fi
+
+# @latest release
+elif [[ "$CIRCLE_BRANCH" == "release" ]]; then
+  # create release commit and release tags
+  npx lerna version --exact --conventional-commits --conventional-graduate --yes --no-push --include-merged-tags --message "chore(release): Publish latest [ci skip]"
+
+  # publish versions that were just computed
+  lernaPublishExitOnFailure from-git --yes --no-push
+
+  if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
+    echo "Published packages to verdaccio"
+    echo "Exiting without pushing release commit or release tags"
+    exit 0
+  fi
+
+  # push release commit
+  git push origin "$CIRCLE_BRANCH"
+
+  # push release tags
+  git tag --points-at HEAD | xargs git push origin
+
+  # fast forward main to release
+  git fetch origin main
+  git checkout main
+  git merge release --ff-only
+  git push origin main
+
+  # fast forward hotfix to release
+  git fetch origin hotfix
+  git checkout hotfix
+  git merge release --ff-only
+  git push origin hotfix
+
+# release candidate or local publish for testing / building binary
+elif [[ "$CIRCLE_BRANCH" =~ ^run-e2e-with-rc\/.* ]] || [[ "$CIRCLE_BRANCH" =~ ^release_rc\/.* ]] || [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
+
+  # force @aws-amplify/cli-internal to be versioned in case this pipeline run does not have any commits that modify the CLI packages
+  if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
+    force_publish_local_args="--force-publish '@aws-amplify/cli-internal'"
+  fi
+  # create release commit and release tags
+  git checkout codebuild-poc && npx lerna version --preid=rc.$CODEBUILD_RESOLVED_SOURCE_VERSION --exact --conventional-prerelease --conventional-commits --yes --no-push --include-merged-tags --message "chore(release): Publish rc [ci skip]" $(echo $force_publish_local_args) --no-commit-hooks
+
+  # if publishing locally to verdaccio
+  if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
+    # publish to verdaccio with no dist tag (default to latest)
+    lernaPublishExitOnFailure from-package --git-head $CODEBUILD_RESOLVED_SOURCE_VERSION --yes --no-push
+    echo "Published packages to verdaccio"
+    echo "Exiting without pushing release commit or release tags"
+    exit 0
+  fi
+
+  # publish versions that were just computed
+  lernaPublishExitOnFailure from-package --git-head $CODEBUILD_RESOLVED_SOURCE_VERSION --yes --no-push --dist-tag rc
+
+  # push release commit
+  git push origin "$CIRCLE_BRANCH"
+
+  # push release tags
+  git tag --points-at HEAD | xargs git push origin
+else
+  echo "branch name" "$CIRCLE_BRANCH" "did not match any branch publish rules. Skipping publish"
+fi

codebuild_specs/base.yml

@@ -0,0 +1,70 @@
+version: 0.2
+env:
+  shell: bash
+  compute-type: BUILD_GENERAL1_MEDIUM
+  imagePullCredentialsType: SERVICE_ROLE
+  variables:
+    CLI_REGION: us-east-1
+    TEST_SUITE: src/__tests__/auth_2b.test.ts
+    AMPLIFY_DIR: '$CODEBUILD_SRC_DIR/out'
+    AMPLIFY_PATH: '$CODEBUILD_SRC_DIR/out/amplify-pkg-linux-x64'
+  # secrets-manager:
+  #   S3_ACCESS_KEY: "secretname" # s3_access_key in secret manager
+
+batch:
+  fast-fail: true
+  build-graph:
+    - identifier: build_linux
+      buildspec: codebuild_specs/build_linux.yml
+      debug-session: true
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+        variables:
+          IS_AMPLIFY_CI: true
+    - identifier: test
+      buildspec: codebuild_specs/test.yml
+      debug-session: true
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+        variables:
+          EXAMPLE_SAMPLE: 'test'
+      depend-on:
+        - build_linux
+    - identifier: validate_cdk_version
+      buildspec: codebuild_specs/validate_cdk_version.yml
+      debug-session: true
+      depend-on:
+        - build_linux
+    - identifier: verify_api_extract
+      buildspec: codebuild_specs/verify_api_extract.yml
+      debug-session: true
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+      depend-on:
+        - build_linux
+    - identifier: verify_yarn_lock
+      buildspec: codebuild_specs/verify_yarn_lock.yml
+      debug-session: true
+      depend-on:
+        - build_linux
+    - identifier: publish_to_local_registry
+      buildspec: codebuild_specs/publish_to_local_registry.yml
+      debug-session: true
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+      depend-on:
+        - build_linux
+    - identifier: build_pkg_binaries_linux
+      buildspec: codebuild_specs/build_pkg_binaries_linux.yml
+      debug-session: true
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+      depend-on:
+        - publish_to_local_registry
+    - identifier: run_e2e_tests
+      buildspec: codebuild_specs/run_e2e_tests.yml
+      debug-session: true
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+      depend-on:
+        - build_pkg_binaries_linux

codebuild_specs/build_linux.yml

@@ -0,0 +1,11 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - source ./shared-scripts.sh && _buildLinux
+
+artifacts:
+  files:
+    - 'shared-scripts.sh'

codebuild_specs/build_pkg_binaries_linux.yml

@@ -0,0 +1,11 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - source ./shared-scripts.sh && _buildBinaries
+
+artifacts:
+  files:
+    - 'shared-scripts.sh'

codebuild_specs/cleanup_resources.yml

@@ -0,0 +1,7 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - echo cleanup running

codebuild_specs/integration_test.yml

@@ -0,0 +1,7 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - echo integration testing

codebuild_specs/lint.yml

@@ -0,0 +1,7 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - source ./shared-scripts.sh && _lint

codebuild_specs/mock_e2e_tests.yml

@@ -0,0 +1,7 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - source ./shared-scripts.sh && _mockE2ETests

codebuild_specs/publish_to_local_registry.yml

@@ -0,0 +1,11 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - source ./shared-scripts.sh && source ./.circleci/local_publish_helpers.sh && _publishToLocalRegistry
+
+artifacts:
+  files:
+    - 'shared-scripts.sh'

codebuild_specs/run_e2e_tests.yml

@@ -0,0 +1,7 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - source ./shared-scripts.sh && _runE2ETestsLinux

codebuild_specs/test.yml

@@ -0,0 +1,7 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - source ./shared-scripts.sh && _testLinux

codebuild_specs/validate_cdk_version.yml

@@ -0,0 +1,7 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - source ./shared-scripts.sh && _validateCDKVersion

codebuild_specs/verify_api_extract.yml

@@ -0,0 +1,7 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - source ./shared-scripts.sh && _verifyAPIExtract

codebuild_specs/verify_yarn_lock.yml

@@ -0,0 +1,7 @@
+version: 0.2
+env:
+  shell: bash
+phases:
+  build:
+    commands:
+      - source ./shared-scripts.sh && _verifyYarnLock