chore: enable rosetta cache for builds

build.sh

@@ -6,11 +6,10 @@ runtarget="build"
 run_tests="true"
 check_prereqs="true"
 check_compat="true"
-extract_snippets="false"
 while [[ "${1:-}" != "" ]]; do
     case $1 in
         -h|--help)
-            echo "Usage: build.sh [--no-bail] [--force|-f] [--skip-test] [--skip-prereqs] [--skip-compat] [--extract]"
+            echo "Usage: build.sh [--no-bail] [--force|-f] [--skip-test] [--skip-prereqs] [--skip-compat]"
             exit 1
             ;;
         --no-bail)
@@ -28,9 +27,6 @@ while [[ "${1:-}" != "" ]]; do
         --skip-compat)
             check_compat="false"
             ;;
-        --extract)
-            extract_snippets="true"
-            ;;
         *)
             echo "Unrecognized parameter: $1"
             exit 1
@@ -81,9 +77,6 @@ trap "rm -rf $MERKLE_BUILD_CACHE" EXIT
 if [ "$run_tests" == "true" ]; then
     runtarget="$runtarget+test"
 fi
-if [ "$extract_snippets" == "true" ]; then
-    runtarget="$runtarget+extract"
-fi
 
 echo "============================================================================================="
 echo "building..."

buildspec-pr.yaml

@@ -16,8 +16,14 @@ phases:
       # Packing the mono-libraries (monocdk & aws-cdk-lib) can cause
       # memory errors. Increasing this value allows our build to more consistently succeed
       - (command -v sysctl || yum install -y procps-ng) && /sbin/sysctl -w vm.max_map_count=2251954
+  pre_build:
+    commands:
+      - /bin/bash ./scripts/cache-load.sh
   build:
     commands:
-      - /bin/bash ./build.sh --extract
-      - /bin/bash ./scripts/transform.sh --extract
+      - /bin/bash ./build.sh
+      - /bin/bash ./scripts/transform.sh
+      # After compilation, run Rosetta (using the cache if available).
+      # This will print errors, and fail the build if there are compilation errors in any packages marked as 'strict'.
+      - /bin/bash ./scripts/run-rosetta.sh
       - git diff-index --exit-code --ignore-space-at-eol --stat HEAD

buildspec.yaml

@@ -16,6 +16,9 @@ phases:
       # Packing the mono-libraries (monocdk & aws-cdk-lib) can cause
       # memory errors. Increasing this value allows our build to more consistently succeed
       - /sbin/sysctl -w vm.max_map_count=2251954
+  pre_build:
+    commands:
+      - /bin/bash ./scripts/cache-load.sh
   build:
     commands:
       - 'if ${BUMP_CANDIDATE:-false}; then /bin/bash ./scripts/bump-candidate.sh; fi'
@@ -25,6 +28,7 @@ phases:
   post_build:
     commands:
       - "[ -f .BUILD_COMPLETED ] && /bin/bash ./pack.sh"
+      - /bin/bash ./scripts/cache-store.sh
 artifacts:
   files:
     - "**/*"

pack.sh

@@ -39,11 +39,7 @@ function lerna_scopes() {
 # Compile examples with respect to "decdk" directory, as all packages will
 # be symlinked there so they can all be included.
 echo "Extracting code samples" >&2
-node --experimental-worker $(which $ROSETTA) \
-  --compile \
-  --output samples.tabl.json \
-  --directory packages/decdk \
-  $(cat $TMPDIR/jsii.txt)
+scripts/run-rosetta.sh $TMPDIR/jsii.txt
 
 # Jsii packaging (all at once using jsii-pacmak)
 echo "Packaging jsii modules" >&2

scripts/cache-load.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# If the environment variable S3_BUILD_CACHE is set, download and extract the
+# tarball it points to into the directory $HOME/.s3buildcache.
+#
+set -eu
+
+cachedir=$HOME/.s3buildcache
+mkdir -p $cachedir
+
+if [[ "${S3_BUILD_CACHE:-}" = "" ]]; then
+    exit 0
+fi
+
+echo "🧳 Build cache enabled: ${S3_BUILD_CACHE}"
+if ! aws s3 ls ${S3_BUILD_CACHE} > /dev/null; then
+    echo "🧳⚠️  Cache not found."
+    exit 0
+fi
+
+if ! (cd $cachedir && aws s3 cp ${S3_BUILD_CACHE} - | tar xzv); then
+    echo "🧳⚠️  Something went wrong fetching the cache. Continuing anyway."
+fi

scripts/cache-store.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# If the environment variable S3_BUILD_CACHE is set, compress and upload the
+# contents of $HOME/.s3buildcache to it.
+#
+set -eu
+
+cachedir=$HOME/.s3buildcache
+mkdir -p $cachedir
+
+if [[ "${S3_BUILD_CACHE:-}" = "" ]]; then
+    exit 0
+fi
+
+echo "🧳 Storing build cache at: ${S3_BUILD_CACHE}"
+
+if ! (cd $cachedir && tar czv . | aws s3 cp - ${S3_BUILD_CACHE}); then
+    echo "🧳⚠️  Something went wrong storing the cache."
+fi
+
+echo "🧳 Finished."

scripts/list-packages

@@ -12,7 +12,9 @@ if (process.argv.length < 4) {
   process.exit(1);
 }
 
-child_process.exec('lerna ls --toposort --json', { shell: true }, (error, stdout) => {
+const lerna = path.resolve(__dirname, '..', 'node_modules', '.bin', 'lerna');
+
+child_process.exec(`${lerna} ls --toposort --json`, { shell: true }, (error, stdout) => {
   if (error) {
     console.error('Error: ', error);
     process.exit(-1);

scripts/run-rosetta.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Run jsii-rosetta on all jsii packages, using the S3 build cache if available.
+#
+#       Usage: run-rosetta [PKGSFILE]
+#
+# If you already have a file with a list of all the JSII package directories
+# in it, pass it as the first argument. Otherwise, this script will run
+# 'list-packages' to determine a list itself.
+set -eu
+scriptdir=$(cd $(dirname $0) && pwd)
+
+ROSETTA=${ROSETTA:-npx jsii-rosetta}
+
+if [[ "${1:-}" = "" ]]; then
+    echo "Collecting package list..." >&2
+    TMPDIR=${TMPDIR:-$(dirname $(mktemp -u))}
+    node $scriptdir/list-packages $TMPDIR/jsii.txt $TMPDIR/nonjsii.txt
+    jsii_pkgs_file=$TMPDIR/jsii.txt
+else
+    jsii_pkgs_file=$1
+fi
+
+rosetta_cache_file=$HOME/.s3buildcache/rosetta-cache.tabl.json
+rosetta_cache_opts=""
+if [[ -f $rosetta_cache_file ]]; then
+    rosetta_cache_opts="--cache-from ${rosetta_cache_file}"
+fi
+
+$ROSETTA \
+  --compile \
+  --output samples.tabl.json \
+  $rosetta_cache_opts \
+  --directory packages/decdk \
+  $(cat $jsii_pkgs_file)
+
+if [[ -d $(dirname $rosetta_cache_file) ]]; then
+    # If the cache directory is available, copy the current tablet into it
+    cp samples.tabl.json $rosetta_cache_file
+fi

scripts/transform.sh

@@ -25,12 +25,11 @@ createSymlinks() {
 
 runtarget="build"
 run_tests="true"
-extract_snippets="false"
 skip_build=""
 while [[ "${1:-}" != "" ]]; do
     case $1 in
         -h|--help)
-            echo "Usage: transform.sh [--skip-test/build] [--extract]"
+            echo "Usage: transform.sh [--skip-test/build]"
             exit 1
             ;;
         --skip-test|--skip-tests)
@@ -39,9 +38,6 @@ while [[ "${1:-}" != "" ]]; do
         --skip-build)
             skip_build="true"
             ;;
-        --extract)
-            extract_snippets="true"
-            ;;
         *)
             echo "Unrecognized options: $1"
             exit 1
@@ -52,9 +48,6 @@ done
 if [ "$run_tests" == "true" ]; then
   runtarget="$runtarget+test"
 fi
-if [ "$extract_snippets" == "true" ]; then
-  runtarget="$runtarget+extract"
-fi
 
 export NODE_OPTIONS="--max-old-space-size=4096 --experimental-worker ${NODE_OPTIONS:-}"