diff --git a/.circleci/publish-codebuild.sh b/.circleci/publish-codebuild.sh new file mode 100755 index 00000000000..1b384577761 --- /dev/null +++ b/.circleci/publish-codebuild.sh @@ -0,0 +1,119 @@ +#!/bin/bash -e + +# lerna has a bug (https://github.com/lerna/lerna/issues/1066) where failed publishes do not set the exit code properly +# this causes the script to keep running even after failed publishes +# this function forces failed publishes to exit on failure +function lernaPublishExitOnFailure { + # exit on failure + set -e + # run lerna publish with the args that were passed to this function + # duplicate stdout to a temp file + # grep the temp file for the lerna err token and return exit 1 if found (-v option inverts grep exit code) + npx lerna publish "$@" | tee /tmp/publish-results && grep -qvz "lerna ERR!" < /tmp/publish-results +} + +if [ -z "$GITHUB_EMAIL" ]; then + if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then + git config --global user.email not@used.com + else + echo "GITHUB_EMAIL email is missing" + exit 1 + fi +else + git config --global user.email $GITHUB_EMAIL +fi + +if [ -z "$GITHUB_USER" ]; then + if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then + git config --global user.name "Doesnt Matter" + else + echo "GITHUB_USER email is missing" + exit 1 + fi +else + git config --global user.name $GITHUB_USER +fi + +if [[ "$CIRCLE_BRANCH" =~ ^tagged-release ]]; then + if [[ "$CIRCLE_BRANCH" =~ ^tagged-release-without-e2e-tests\/.* ]]; then + # Remove tagged-release-without-e2e-tests/ + export NPM_TAG="${CIRCLE_BRANCH/tagged-release-without-e2e-tests\//}" + elif [[ "$CIRCLE_BRANCH" =~ ^tagged-release\/.* ]]; then + # Remove tagged-release/ + export NPM_TAG="${CIRCLE_BRANCH/tagged-release\//}" + fi + if [ -z "$NPM_TAG" ]; then + echo "Tag name is missing. Name your branch with either tagged-release/ or tagged-release-without-e2e-tests/" + exit 1 + fi + + if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then + echo "Publishing to local registry under latest tag" + lernaPublishExitOnFailure --exact --preid=$NPM_TAG --conventional-commits --conventional-prerelease --no-push --yes --include-merged-tags + else + echo "Publishing to NPM under $NPM_TAG tag" + lernaPublishExitOnFailure --exact --dist-tag=$NPM_TAG --preid=$NPM_TAG --conventional-commits --conventional-prerelease --message "chore(release): Publish tagged release $NPM_TAG [ci skip]" --yes --include-merged-tags + fi + +# @latest release +elif [[ "$CIRCLE_BRANCH" == "release" ]]; then + # create release commit and release tags + npx lerna version --exact --conventional-commits --conventional-graduate --yes --no-push --include-merged-tags --message "chore(release): Publish latest [ci skip]" + + # publish versions that were just computed + lernaPublishExitOnFailure from-git --yes --no-push + + if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then + echo "Published packages to verdaccio" + echo "Exiting without pushing release commit or release tags" + exit 0 + fi + + # push release commit + git push origin "$CIRCLE_BRANCH" + + # push release tags + git tag --points-at HEAD | xargs git push origin + + # fast forward main to release + git fetch origin main + git checkout main + git merge release --ff-only + git push origin main + + # fast forward hotfix to release + git fetch origin hotfix + git checkout hotfix + git merge release --ff-only + git push origin hotfix + +# release candidate or local publish for testing / building binary +elif [[ "$CIRCLE_BRANCH" =~ ^run-e2e-with-rc\/.* ]] || [[ "$CIRCLE_BRANCH" =~ ^release_rc\/.* ]] || [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then + + # force @aws-amplify/cli-internal to be versioned in case this pipeline run does not have any commits that modify the CLI packages + if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then + force_publish_local_args="--force-publish '@aws-amplify/cli-internal'" + fi + # create release commit and release tags + git checkout codebuild-poc && npx lerna version --preid=rc.$CODEBUILD_RESOLVED_SOURCE_VERSION --exact --conventional-prerelease --conventional-commits --yes --no-push --include-merged-tags --message "chore(release): Publish rc [ci skip]" $(echo $force_publish_local_args) --no-commit-hooks + + # if publishing locally to verdaccio + if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then + # publish to verdaccio with no dist tag (default to latest) + lernaPublishExitOnFailure from-package --git-head $CODEBUILD_RESOLVED_SOURCE_VERSION --yes --no-push + echo "Published packages to verdaccio" + echo "Exiting without pushing release commit or release tags" + exit 0 + fi + + # publish versions that were just computed + lernaPublishExitOnFailure from-package --git-head $CODEBUILD_RESOLVED_SOURCE_VERSION --yes --no-push --dist-tag rc + + # push release commit + git push origin "$CIRCLE_BRANCH" + + # push release tags + git tag --points-at HEAD | xargs git push origin +else + echo "branch name" "$CIRCLE_BRANCH" "did not match any branch publish rules. Skipping publish" +fi diff --git a/codebuild_specs/base.yml b/codebuild_specs/base.yml new file mode 100644 index 00000000000..88226963127 --- /dev/null +++ b/codebuild_specs/base.yml @@ -0,0 +1,70 @@ +version: 0.2 +env: + shell: bash + compute-type: BUILD_GENERAL1_MEDIUM + imagePullCredentialsType: SERVICE_ROLE + variables: + CLI_REGION: us-east-1 + TEST_SUITE: src/__tests__/auth_2b.test.ts + AMPLIFY_DIR: '$CODEBUILD_SRC_DIR/out' + AMPLIFY_PATH: '$CODEBUILD_SRC_DIR/out/amplify-pkg-linux-x64' + # secrets-manager: + # S3_ACCESS_KEY: "secretname" # s3_access_key in secret manager + +batch: + fast-fail: true + build-graph: + - identifier: build_linux + buildspec: codebuild_specs/build_linux.yml + debug-session: true + env: + compute-type: BUILD_GENERAL1_LARGE + variables: + IS_AMPLIFY_CI: true + - identifier: test + buildspec: codebuild_specs/test.yml + debug-session: true + env: + compute-type: BUILD_GENERAL1_LARGE + variables: + EXAMPLE_SAMPLE: 'test' + depend-on: + - build_linux + - identifier: validate_cdk_version + buildspec: codebuild_specs/validate_cdk_version.yml + debug-session: true + depend-on: + - build_linux + - identifier: verify_api_extract + buildspec: codebuild_specs/verify_api_extract.yml + debug-session: true + env: + compute-type: BUILD_GENERAL1_LARGE + depend-on: + - build_linux + - identifier: verify_yarn_lock + buildspec: codebuild_specs/verify_yarn_lock.yml + debug-session: true + depend-on: + - build_linux + - identifier: publish_to_local_registry + buildspec: codebuild_specs/publish_to_local_registry.yml + debug-session: true + env: + compute-type: BUILD_GENERAL1_LARGE + depend-on: + - build_linux + - identifier: build_pkg_binaries_linux + buildspec: codebuild_specs/build_pkg_binaries_linux.yml + debug-session: true + env: + compute-type: BUILD_GENERAL1_LARGE + depend-on: + - publish_to_local_registry + - identifier: run_e2e_tests + buildspec: codebuild_specs/run_e2e_tests.yml + debug-session: true + env: + compute-type: BUILD_GENERAL1_LARGE + depend-on: + - build_pkg_binaries_linux diff --git a/codebuild_specs/build_linux.yml b/codebuild_specs/build_linux.yml new file mode 100644 index 00000000000..273b055f828 --- /dev/null +++ b/codebuild_specs/build_linux.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _buildLinux + +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/build_pkg_binaries_linux.yml b/codebuild_specs/build_pkg_binaries_linux.yml new file mode 100644 index 00000000000..eb1c24c5013 --- /dev/null +++ b/codebuild_specs/build_pkg_binaries_linux.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _buildBinaries + +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/cleanup_resources.yml b/codebuild_specs/cleanup_resources.yml new file mode 100644 index 00000000000..c9249fc7446 --- /dev/null +++ b/codebuild_specs/cleanup_resources.yml @@ -0,0 +1,7 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - echo cleanup running diff --git a/codebuild_specs/integration_test.yml b/codebuild_specs/integration_test.yml new file mode 100644 index 00000000000..d2e6df4b7a0 --- /dev/null +++ b/codebuild_specs/integration_test.yml @@ -0,0 +1,7 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - echo integration testing diff --git a/codebuild_specs/lint.yml b/codebuild_specs/lint.yml new file mode 100644 index 00000000000..7363c6c10d0 --- /dev/null +++ b/codebuild_specs/lint.yml @@ -0,0 +1,7 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _lint diff --git a/codebuild_specs/mock_e2e_tests.yml b/codebuild_specs/mock_e2e_tests.yml new file mode 100644 index 00000000000..15573725d6a --- /dev/null +++ b/codebuild_specs/mock_e2e_tests.yml @@ -0,0 +1,7 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _mockE2ETests diff --git a/codebuild_specs/publish_to_local_registry.yml b/codebuild_specs/publish_to_local_registry.yml new file mode 100644 index 00000000000..365d603f745 --- /dev/null +++ b/codebuild_specs/publish_to_local_registry.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && source ./.circleci/local_publish_helpers.sh && _publishToLocalRegistry + +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/run_e2e_tests.yml b/codebuild_specs/run_e2e_tests.yml new file mode 100644 index 00000000000..76989a00309 --- /dev/null +++ b/codebuild_specs/run_e2e_tests.yml @@ -0,0 +1,7 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _runE2ETestsLinux diff --git a/codebuild_specs/test.yml b/codebuild_specs/test.yml new file mode 100644 index 00000000000..ed813463ab0 --- /dev/null +++ b/codebuild_specs/test.yml @@ -0,0 +1,7 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _testLinux diff --git a/codebuild_specs/validate_cdk_version.yml b/codebuild_specs/validate_cdk_version.yml new file mode 100644 index 00000000000..f1bf130dbc6 --- /dev/null +++ b/codebuild_specs/validate_cdk_version.yml @@ -0,0 +1,7 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _validateCDKVersion diff --git a/codebuild_specs/verify_api_extract.yml b/codebuild_specs/verify_api_extract.yml new file mode 100644 index 00000000000..ac360de6cfd --- /dev/null +++ b/codebuild_specs/verify_api_extract.yml @@ -0,0 +1,7 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _verifyAPIExtract diff --git a/codebuild_specs/verify_yarn_lock.yml b/codebuild_specs/verify_yarn_lock.yml new file mode 100644 index 00000000000..9fba2102f79 --- /dev/null +++ b/codebuild_specs/verify_yarn_lock.yml @@ -0,0 +1,7 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _verifyYarnLock diff --git a/shared-scripts.sh b/shared-scripts.sh new file mode 100644 index 00000000000..59308d5534b --- /dev/null +++ b/shared-scripts.sh @@ -0,0 +1,239 @@ +#!/bin/bash + +# set exit on error to true +set -e + +# We have custom caching for our CodeBuild pipelines +# which allows us to share caches with jobs in the same batch + +# storeCache +function storeCache { + localPath="$1" + alias="$2" + s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" + echo writing cache to $s3Path + # zip contents and upload to s3 + if ! (cd $localPath && tar cz . | aws s3 cp - $s3Path); then + echo Something went wrong storing the cache. + fi + echo done writing cache + cd $CODEBUILD_SRC_DIR +} +function storeCacheFile { + localFilePath="$1" + alias="$2" + s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" + echo writing cache to $s3Path + # zip contents and upload to s3 + if ! (aws s3 cp $localFilePath $s3Path); then + echo Something went wrong storing the cache. + fi + echo done writing cache + cd $CODEBUILD_SRC_DIR +} +# loadCache +function loadCache { + alias="$1" + localPath="$2" + s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" + echo loading cache from $s3Path + # create directory if it doesn't exist yet + mkdir -p $localPath + # check if cache exists in s3 + if ! aws s3 ls $s3Path > /dev/null; then + echo "Cache not found." + exit 0 + fi + # load cache and unzip it + if ! (cd $localPath && aws s3 cp $s3Path - | tar xz); then + echo "Something went wrong fetching the cache. Continuing anyway." + fi + echo done loading cache + cd $CODEBUILD_SRC_DIR +} +function loadCacheFile { + alias="$1" + localFilePath="$2" + s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" + echo loading cache file from $s3Path + # check if cache file exists in s3 + if ! aws s3 ls $s3Path > /dev/null; then + echo "Cache file not found." + exit 0 + fi + # load cache file + if ! (aws s3 cp $s3Path $localFilePath); then + echo "Something went wrong fetching the cache file. Continuing anyway." + fi + echo done loading cache + cd $CODEBUILD_SRC_DIR +} + + + + + +function _setShell { + echo Setting Shell + yarn config set script-shell $(which bash) +} +function _buildLinux { + _setShell + echo Linux Build + yarn run production-build + # copy [repo, ~/.cache, and .ssh to s3] + storeCache $CODEBUILD_SRC_DIR repo + storeCache $HOME/.cache .cache +} +function _buildWindows { + _setShell + echo Windows Build + yarn run production-build + # copy [repo, .cache, and .ssh to s3] + storeCache $CODEBUILD_SRC_DIR repo-windows + storeCache $HOME/.cache .cache-windows +} +function _testLinux { + echo Run Test + # download [repo, .cache from s3] + loadCache repo $CODEBUILD_SRC_DIR + loadCache .cache $HOME/.cache + # run tests + yarn test-ci + echo collecting coverage + yarn coverage +} +function _validateCDKVersion { + echo Validate CDK Version + # download [repo, .cache from s3] + loadCache repo $CODEBUILD_SRC_DIR + loadCache .cache $HOME/.cache + yarn ts-node .circleci/validate_cdk_version.ts +} +function _lint { + echo Linting + # download [repo, .cache from s3] + loadCache repo $CODEBUILD_SRC_DIR + loadCache .cache $HOME/.cache + + yarn lint-check + yarn lint-check-package-json + yarn prettier-check +} +function _verifyAPIExtract { + echo Verify API Extract + + # download [repo, .cache from s3] + loadCache repo $CODEBUILD_SRC_DIR + loadCache .cache $HOME/.cache + + yarn verify-api-extract +} +function _verifyYarnLock { + echo "Verify Yarn Lock" + + # download [repo, .cache from s3] + loadCache repo $CODEBUILD_SRC_DIR + loadCache .cache $HOME/.cache + + yarn verify-yarn-lock +} +function _verifyVersionsMatch { + echo Verify Versions Match + + # download [repo, .cache, verdaccio-cache from s3] + loadCache repo $CODEBUILD_SRC_DIR + loadCache .cache $HOME/.cache + loadCache verdaccio-cache $HOME/verdaccio-cache + + source .circleci/local_publish_helpers.sh + startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + setNpmRegistryUrlToLocal + changeNpmGlobalPath + checkPackageVersionsInLocalNpmRegistry +} +function _mockE2ETests { + # download [repo, .cache from s3] + loadCache repo $CODEBUILD_SRC_DIR + loadCache .cache $HOME/.cache + + source .circleci/local_publish_helpers.sh + cd packages/amplify-util-mock/ + yarn e2e +} +function _publishToLocalRegistry { + echo "Publish To Local Registry" + + # download [repo, .cache from s3] + loadCache repo $CODEBUILD_SRC_DIR + loadCache .cache $HOME/.cache + + startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + setNpmRegistryUrlToLocal + export LOCAL_PUBLISH_TO_LATEST=true + ./.circleci/publish-codebuild.sh + unsetNpmRegistryUrl + + echo Generate Change Log + git reset --soft HEAD~1 + yarn ts-node scripts/unified-changelog.ts + cat UNIFIED_CHANGELOG.md + + echo Save new amplify Github tag + node scripts/echo-current-cli-version.js > .amplify-pkg-version + + # copy [verdaccio-cache, changelog, pkgtag to s3] + storeCache $HOME/verdaccio-cache verdaccio-cache + storeCacheFile $CODEBUILD_SRC_DIR/UNIFIED_CHANGELOG.md UNIFIED_CHANGELOG.md + storeCacheFile $CODEBUILD_SRC_DIR/.amplify-pkg-version .amplify-pkg-version +} +function _uploadPkgBinaries { + # download [repo, pkg-binaries, from s3] + echo Consolidate binaries cache and upload + source .circleci/local_publish_helpers.sh + uploadPkgCli + # copy [repo/out to s3] +} +function _buildBinaries { + echo Start verdaccio and package CLI + # download [repo, yarn, verdaccio from s3] + loadCache repo $CODEBUILD_SRC_DIR + loadCache .cache $HOME/.cache + loadCache verdaccio-cache $HOME/verdaccio-cache + loadCacheFile .amplify-pkg-version $CODEBUILD_SRC_DIR/.amplify-pkg-version + loadCacheFile UNIFIED_CHANGELOG.md $CODEBUILD_SRC_DIR/UNIFIED_CHANGELOG.md + + source .circleci/local_publish_helpers.sh + startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + setNpmRegistryUrlToLocal + changeNpmGlobalPath + generatePkgCli linux + # generatePkgCli macos + # generatePkgCli win + # generatePkgCli arm + + unsetNpmRegistryUrl + + # copy [repo/out to s3] + storeCache $CODEBUILD_SRC_DIR/out repo-out +} +function _runE2ETestsLinux { + echo RUN E2E Tests + + loadCache repo $CODEBUILD_SRC_DIR + loadCache .cache $HOME/.cache + loadCache verdaccio-cache $HOME/verdaccio-cache + loadCache repo-out $CODEBUILD_SRC_DIR/out + loadCacheFile .amplify-pkg-version $CODEBUILD_SRC_DIR/.amplify-pkg-version + loadCacheFile UNIFIED_CHANGELOG.md $CODEBUILD_SRC_DIR/UNIFIED_CHANGELOG.md + + source .circleci/local_publish_helpers.sh + source $BASH_ENV + startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + setNpmRegistryUrlToLocal + changeNpmGlobalPath + amplify version + #should just fail here.. + cd packages/amplify-e2e-tests + retry runE2eTest +}