Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: enable rosetta cache for builds #17336

Merged
merged 5 commits into from
Nov 5, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,10 @@ runtarget="build"
run_tests="true"
check_prereqs="true"
check_compat="true"
extract_snippets="false"
while [[ "${1:-}" != "" ]]; do
case $1 in
-h|--help)
echo "Usage: build.sh [--no-bail] [--force|-f] [--skip-test] [--skip-prereqs] [--skip-compat] [--extract]"
echo "Usage: build.sh [--no-bail] [--force|-f] [--skip-test] [--skip-prereqs] [--skip-compat]"
exit 1
;;
--no-bail)
Expand All @@ -28,9 +27,6 @@ while [[ "${1:-}" != "" ]]; do
--skip-compat)
check_compat="false"
;;
--extract)
extract_snippets="true"
;;
*)
echo "Unrecognized parameter: $1"
exit 1
Expand Down Expand Up @@ -81,9 +77,6 @@ trap "rm -rf $MERKLE_BUILD_CACHE" EXIT
if [ "$run_tests" == "true" ]; then
runtarget="$runtarget+test"
fi
if [ "$extract_snippets" == "true" ]; then
runtarget="$runtarget+extract"
fi

echo "============================================================================================="
echo "building..."
Expand Down
10 changes: 8 additions & 2 deletions buildspec-pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,14 @@ phases:
# Packing the mono-libraries (monocdk & aws-cdk-lib) can cause
# memory errors. Increasing this value allows our build to more consistently succeed
- (command -v sysctl || yum install -y procps-ng) && /sbin/sysctl -w vm.max_map_count=2251954
pre_build:
commands:
- /bin/bash ./scripts/cache-load.sh
build:
commands:
- /bin/bash ./build.sh --extract
- /bin/bash ./scripts/transform.sh --extract
- /bin/bash ./build.sh
- /bin/bash ./scripts/transform.sh
# After compilation, run Rosetta (using the cache if available).
# This will print errors, and fail the build if there are compilation errors in any packages marked as 'strict'.
- /bin/bash ./scripts/run-rosetta.sh
- git diff-index --exit-code --ignore-space-at-eol --stat HEAD
4 changes: 4 additions & 0 deletions buildspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ phases:
# Packing the mono-libraries (monocdk & aws-cdk-lib) can cause
# memory errors. Increasing this value allows our build to more consistently succeed
- /sbin/sysctl -w vm.max_map_count=2251954
pre_build:
commands:
- /bin/bash ./scripts/cache-load.sh
build:
commands:
- 'if ${BUMP_CANDIDATE:-false}; then /bin/bash ./scripts/bump-candidate.sh; fi'
Expand All @@ -25,6 +28,7 @@ phases:
post_build:
commands:
- "[ -f .BUILD_COMPLETED ] && /bin/bash ./pack.sh"
- /bin/bash ./scripts/cache-store.sh
artifacts:
files:
- "**/*"
Expand Down
6 changes: 1 addition & 5 deletions pack.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,7 @@ function lerna_scopes() {
# Compile examples with respect to "decdk" directory, as all packages will
# be symlinked there so they can all be included.
echo "Extracting code samples" >&2
node --experimental-worker $(which $ROSETTA) \
--compile \
--output samples.tabl.json \
--directory packages/decdk \
$(cat $TMPDIR/jsii.txt)
scripts/run-rosetta.sh $TMPDIR/jsii.txt

# Jsii packaging (all at once using jsii-pacmak)
echo "Packaging jsii modules" >&2
Expand Down
23 changes: 23 additions & 0 deletions scripts/cache-load.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/bin/bash
#
# If the environment variable S3_BUILD_CACHE is set, download and extract the
# tarball it points to into the directory $HOME/.s3buildcache.
#
set -eu

cachedir=$HOME/.s3buildcache
mkdir -p $cachedir
Comment on lines +8 to +9
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

~ (Nit) - Move this to after the env variable check so this script no-ops if the environment variable isn't set.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually I did this on purpose. It's convenient to always be able to rely on the ~/.s3buildcache directory existing (whether it's empty or not).

Makes the shell scripts easier to write... (?)


if [[ "${S3_BUILD_CACHE:-}" = "" ]]; then
exit 0
fi

echo "🧳 Build cache enabled: ${S3_BUILD_CACHE}"
if ! aws s3 ls ${S3_BUILD_CACHE} > /dev/null; then
echo "🧳⚠️ Cache not found."
exit 0
fi

if ! (cd $cachedir && aws s3 cp ${S3_BUILD_CACHE} - | tar xzv); then
echo "🧳⚠️ Something went wrong fetching the cache. Continuing anyway."
fi
21 changes: 21 additions & 0 deletions scripts/cache-store.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/bash
#
# If the environment variable S3_BUILD_CACHE is set, compress and upload the
# contents of $HOME/.s3buildcache to it.
#
set -eu

cachedir=$HOME/.s3buildcache
mkdir -p $cachedir

if [[ "${S3_BUILD_CACHE:-}" = "" ]]; then
exit 0
fi

echo "🧳 Storing build cache at: ${S3_BUILD_CACHE}"

if ! (cd $cachedir && tar czv . | aws s3 cp - ${S3_BUILD_CACHE}); then
echo "🧳⚠️ Something went wrong storing the cache."
fi

echo "🧳 Finished."
4 changes: 3 additions & 1 deletion scripts/list-packages
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ if (process.argv.length < 4) {
process.exit(1);
}

child_process.exec('lerna ls --toposort --json', { shell: true }, (error, stdout) => {
const lerna = path.resolve(__dirname, '..', 'node_modules', '.bin', 'lerna');

child_process.exec(`${lerna} ls --toposort --json`, { shell: true }, (error, stdout) => {
if (error) {
console.error('Error: ', error);
process.exit(-1);
Expand Down
40 changes: 40 additions & 0 deletions scripts/run-rosetta.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/bin/bash
#
# Run jsii-rosetta on all jsii packages, using the S3 build cache if available.
#
# Usage: run-rosetta [PKGSFILE]
#
# If you already have a file with a list of all the JSII package directories
# in it, pass it as the first argument. Otherwise, this script will run
# 'list-packages' to determine a list itself.
set -eu
scriptdir=$(cd $(dirname $0) && pwd)

ROSETTA=${ROSETTA:-npx jsii-rosetta}

if [[ "${1:-}" = "" ]]; then
echo "Collecting package list..." >&2
TMPDIR=${TMPDIR:-$(dirname $(mktemp -u))}
node $scriptdir/list-packages $TMPDIR/jsii.txt $TMPDIR/nonjsii.txt
jsii_pkgs_file=$TMPDIR/jsii.txt
else
jsii_pkgs_file=$1
fi

rosetta_cache_file=$HOME/.s3buildcache/rosetta-cache.tabl.json
rosetta_cache_opts=""
if [[ -f $rosetta_cache_file ]]; then
rosetta_cache_opts="--cache-from ${rosetta_cache_file}"
fi

$ROSETTA \
--compile \
--output samples.tabl.json \
$rosetta_cache_opts \
--directory packages/decdk \
$(cat $jsii_pkgs_file)

if [[ -d $(dirname $rosetta_cache_file) ]]; then
# If the cache directory is available, copy the current tablet into it
cp samples.tabl.json $rosetta_cache_file
fi
9 changes: 1 addition & 8 deletions scripts/transform.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,11 @@ createSymlinks() {

runtarget="build"
run_tests="true"
extract_snippets="false"
skip_build=""
while [[ "${1:-}" != "" ]]; do
case $1 in
-h|--help)
echo "Usage: transform.sh [--skip-test/build] [--extract]"
echo "Usage: transform.sh [--skip-test/build]"
exit 1
;;
--skip-test|--skip-tests)
Expand All @@ -39,9 +38,6 @@ while [[ "${1:-}" != "" ]]; do
--skip-build)
skip_build="true"
;;
--extract)
extract_snippets="true"
;;
*)
echo "Unrecognized options: $1"
exit 1
Expand All @@ -52,9 +48,6 @@ done
if [ "$run_tests" == "true" ]; then
runtarget="$runtarget+test"
fi
if [ "$extract_snippets" == "true" ]; then
runtarget="$runtarget+extract"
fi

export NODE_OPTIONS="--max-old-space-size=4096 --experimental-worker ${NODE_OPTIONS:-}"

Expand Down