diff --git a/build.sh b/build.sh index aae39e94ea730..64ecc043faf45 100755 --- a/build.sh +++ b/build.sh @@ -6,11 +6,10 @@ runtarget="build" run_tests="true" check_prereqs="true" check_compat="true" -extract_snippets="false" while [[ "${1:-}" != "" ]]; do case $1 in -h|--help) - echo "Usage: build.sh [--no-bail] [--force|-f] [--skip-test] [--skip-prereqs] [--skip-compat] [--extract]" + echo "Usage: build.sh [--no-bail] [--force|-f] [--skip-test] [--skip-prereqs] [--skip-compat]" exit 1 ;; --no-bail) @@ -28,9 +27,6 @@ while [[ "${1:-}" != "" ]]; do --skip-compat) check_compat="false" ;; - --extract) - extract_snippets="true" - ;; *) echo "Unrecognized parameter: $1" exit 1 @@ -81,9 +77,6 @@ trap "rm -rf $MERKLE_BUILD_CACHE" EXIT if [ "$run_tests" == "true" ]; then runtarget="$runtarget+test" fi -if [ "$extract_snippets" == "true" ]; then - runtarget="$runtarget+extract" -fi echo "=============================================================================================" echo "building..." diff --git a/buildspec-pr.yaml b/buildspec-pr.yaml index 647b78849b3e8..ade1f4a9be0c6 100644 --- a/buildspec-pr.yaml +++ b/buildspec-pr.yaml @@ -16,8 +16,14 @@ phases: # Packing the mono-libraries (monocdk & aws-cdk-lib) can cause # memory errors. Increasing this value allows our build to more consistently succeed - (command -v sysctl || yum install -y procps-ng) && /sbin/sysctl -w vm.max_map_count=2251954 + pre_build: + commands: + - /bin/bash ./scripts/cache-load.sh build: commands: - - /bin/bash ./build.sh --extract - - /bin/bash ./scripts/transform.sh --extract + - /bin/bash ./build.sh + - /bin/bash ./scripts/transform.sh + # After compilation, run Rosetta (using the cache if available). + # This will print errors, and fail the build if there are compilation errors in any packages marked as 'strict'. + - /bin/bash ./scripts/run-rosetta.sh - git diff-index --exit-code --ignore-space-at-eol --stat HEAD diff --git a/buildspec.yaml b/buildspec.yaml index 0fc990a17c805..94a75e2357f78 100644 --- a/buildspec.yaml +++ b/buildspec.yaml @@ -16,6 +16,9 @@ phases: # Packing the mono-libraries (monocdk & aws-cdk-lib) can cause # memory errors. Increasing this value allows our build to more consistently succeed - /sbin/sysctl -w vm.max_map_count=2251954 + pre_build: + commands: + - /bin/bash ./scripts/cache-load.sh build: commands: - 'if ${BUMP_CANDIDATE:-false}; then /bin/bash ./scripts/bump-candidate.sh; fi' @@ -25,6 +28,7 @@ phases: post_build: commands: - "[ -f .BUILD_COMPLETED ] && /bin/bash ./pack.sh" + - /bin/bash ./scripts/cache-store.sh artifacts: files: - "**/*" diff --git a/pack.sh b/pack.sh index 168a17f972406..81eecafabe187 100755 --- a/pack.sh +++ b/pack.sh @@ -39,11 +39,7 @@ function lerna_scopes() { # Compile examples with respect to "decdk" directory, as all packages will # be symlinked there so they can all be included. echo "Extracting code samples" >&2 -node --experimental-worker $(which $ROSETTA) \ - --compile \ - --output samples.tabl.json \ - --directory packages/decdk \ - $(cat $TMPDIR/jsii.txt) +scripts/run-rosetta.sh $TMPDIR/jsii.txt # Jsii packaging (all at once using jsii-pacmak) echo "Packaging jsii modules" >&2 diff --git a/scripts/cache-load.sh b/scripts/cache-load.sh new file mode 100755 index 0000000000000..23aff254f4f1f --- /dev/null +++ b/scripts/cache-load.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# If the environment variable S3_BUILD_CACHE is set, download and extract the +# tarball it points to into the directory $HOME/.s3buildcache. +# +set -eu + +cachedir=$HOME/.s3buildcache +mkdir -p $cachedir + +if [[ "${S3_BUILD_CACHE:-}" = "" ]]; then + exit 0 +fi + +echo "🧳 Build cache enabled: ${S3_BUILD_CACHE}" +if ! aws s3 ls ${S3_BUILD_CACHE} > /dev/null; then + echo "🧳⚠️ Cache not found." + exit 0 +fi + +if ! (cd $cachedir && aws s3 cp ${S3_BUILD_CACHE} - | tar xzv); then + echo "🧳⚠️ Something went wrong fetching the cache. Continuing anyway." +fi diff --git a/scripts/cache-store.sh b/scripts/cache-store.sh new file mode 100644 index 0000000000000..aaf99cbca4f0f --- /dev/null +++ b/scripts/cache-store.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# If the environment variable S3_BUILD_CACHE is set, compress and upload the +# contents of $HOME/.s3buildcache to it. +# +set -eu + +cachedir=$HOME/.s3buildcache +mkdir -p $cachedir + +if [[ "${S3_BUILD_CACHE:-}" = "" ]]; then + exit 0 +fi + +echo "🧳 Storing build cache at: ${S3_BUILD_CACHE}" + +if ! (cd $cachedir && tar czv . | aws s3 cp - ${S3_BUILD_CACHE}); then + echo "🧳⚠️ Something went wrong storing the cache." +fi + +echo "🧳 Finished." diff --git a/scripts/list-packages b/scripts/list-packages index 95a2c5ecc0379..fe89f84db9410 100755 --- a/scripts/list-packages +++ b/scripts/list-packages @@ -12,7 +12,9 @@ if (process.argv.length < 4) { process.exit(1); } -child_process.exec('lerna ls --toposort --json', { shell: true }, (error, stdout) => { +const lerna = path.resolve(__dirname, '..', 'node_modules', '.bin', 'lerna'); + +child_process.exec(`${lerna} ls --toposort --json`, { shell: true }, (error, stdout) => { if (error) { console.error('Error: ', error); process.exit(-1); diff --git a/scripts/run-rosetta.sh b/scripts/run-rosetta.sh new file mode 100644 index 0000000000000..0148eeccc3630 --- /dev/null +++ b/scripts/run-rosetta.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Run jsii-rosetta on all jsii packages, using the S3 build cache if available. +# +# Usage: run-rosetta [PKGSFILE] +# +# If you already have a file with a list of all the JSII package directories +# in it, pass it as the first argument. Otherwise, this script will run +# 'list-packages' to determine a list itself. +set -eu +scriptdir=$(cd $(dirname $0) && pwd) + +ROSETTA=${ROSETTA:-npx jsii-rosetta} + +if [[ "${1:-}" = "" ]]; then + echo "Collecting package list..." >&2 + TMPDIR=${TMPDIR:-$(dirname $(mktemp -u))} + node $scriptdir/list-packages $TMPDIR/jsii.txt $TMPDIR/nonjsii.txt + jsii_pkgs_file=$TMPDIR/jsii.txt +else + jsii_pkgs_file=$1 +fi + +rosetta_cache_file=$HOME/.s3buildcache/rosetta-cache.tabl.json +rosetta_cache_opts="" +if [[ -f $rosetta_cache_file ]]; then + rosetta_cache_opts="--cache-from ${rosetta_cache_file}" +fi + +$ROSETTA \ + --compile \ + --output samples.tabl.json \ + $rosetta_cache_opts \ + --directory packages/decdk \ + $(cat $jsii_pkgs_file) + +if [[ -d $(dirname $rosetta_cache_file) ]]; then + # If the cache directory is available, copy the current tablet into it + cp samples.tabl.json $rosetta_cache_file +fi diff --git a/scripts/transform.sh b/scripts/transform.sh index da780c3df49ff..c0bb83b3a6edf 100755 --- a/scripts/transform.sh +++ b/scripts/transform.sh @@ -25,12 +25,11 @@ createSymlinks() { runtarget="build" run_tests="true" -extract_snippets="false" skip_build="" while [[ "${1:-}" != "" ]]; do case $1 in -h|--help) - echo "Usage: transform.sh [--skip-test/build] [--extract]" + echo "Usage: transform.sh [--skip-test/build]" exit 1 ;; --skip-test|--skip-tests) @@ -39,9 +38,6 @@ while [[ "${1:-}" != "" ]]; do --skip-build) skip_build="true" ;; - --extract) - extract_snippets="true" - ;; *) echo "Unrecognized options: $1" exit 1 @@ -52,9 +48,6 @@ done if [ "$run_tests" == "true" ]; then runtarget="$runtarget+test" fi -if [ "$extract_snippets" == "true" ]; then - runtarget="$runtarget+extract" -fi export NODE_OPTIONS="--max-old-space-size=4096 --experimental-worker ${NODE_OPTIONS:-}"