From 5637adfb4a6809a18d888693d27e2f632e6eced8 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Thu, 25 May 2023 16:36:55 -0700 Subject: [PATCH] Set up remote caching in CI (cherry-pick of #19144) Runs a sidecar bazel-remote server in a docker container. This server uses S3 as its storage backend, specifically a new cache.pantsbuild.org bucket. Read/write access requires our AWS secret key, which is not accessible to PRs across forks. So to write to this cache, a PR must be from a branch in the pantsbuild/pants repo. We can establish protocols by which a set of trusted contributors can push to feature branches in pantsbuild/pants and submit PRs from them, rather than from forks, to get the best CI speedups. PRs across forks can still read from the cache, via restricted creds. --- .github/workflows/test-cron.yaml | 253 ++++++++++++++++++ .github/workflows/test.yaml | 198 ++++++++++++++ .../bin/generate_github_workflows.py | 68 ++++- 3 files changed, 515 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-cron.yaml b/.github/workflows/test-cron.yaml index fb308ce82c9..af6df6f8abc 100644 --- a/.github/workflows/test-cron.yaml +++ b/.github/workflows/test-cron.yaml @@ -332,6 +332,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -424,6 +447,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -491,6 +537,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -558,6 +627,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -625,6 +717,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -692,6 +807,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -759,6 +897,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -826,6 +987,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -893,6 +1077,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -960,6 +1167,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1027,6 +1257,29 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n\ + \ CACHE_WRITE=false\n # If no secret read/write creds, use hard-coded read-only\ + \ creds, so that\n # cross-fork PRs can at least read from the cache.\n \ + \ # These creds are hard-coded here in this public repo, which makes the bucket\n\ + \ # world-readable. But since putting raw AWS tokens in a public repo, even\n\ + \ # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it\ + \ would be terrible if we were scanned, since this is public\n # on purpose,\ + \ but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo='\ + \ | base64 -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000\ + \ -v ~/bazel-remote:/data -p 9092:9092\ + \ buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" \ + \ --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\" \ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com\ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\"\ + \ >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\ + \necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1b1309e2c7b..885e175ca96 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -606,6 +606,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -743,6 +761,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -809,6 +845,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -875,6 +929,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -941,6 +1013,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1007,6 +1097,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1073,6 +1181,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1139,6 +1265,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1205,6 +1349,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1271,6 +1433,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1337,6 +1517,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + name: Launch bazel-remote + run: "mkdir -p ~/bazel-remote\nif [[ -z \"${AWS_ACCESS_KEY_ID}\" ]]; then\n CACHE_WRITE=false\n # If no secret read/write\ + \ creds, use hard-coded read-only creds, so that\n # cross-fork PRs can at least read from the cache.\n # These\ + \ creds are hard-coded here in this public repo, which makes the bucket\n # world-readable. But since putting raw\ + \ AWS tokens in a public repo, even\n # deliberately, is icky, we base64-them. This will at least help hide from\n\ + \ # automated scanners that look for checked in AWS keys.\n # Not that it would be terrible if we were scanned,\ + \ since this is public\n # on purpose, but it's best not to draw attention.\n AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK'\ + \ | base64 -d)\n AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64\ + \ -d)\nelse\n CACHE_WRITE=true\nfi\ndocker run --detach -u 1001:1000 -v ~/bazel-remote:/data \ + \ -p 9092:9092 buchgr/bazel-remote-cache:v2.4.1 --s3.auth_method=access_key\ + \ --s3.access_key_id=\"${AWS_ACCESS_KEY_ID}\" --s3.secret_access_key=\"${AWS_SECRET_ACCESS_KEY}\"\ + \ --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com \ + \ --max_size 30\necho \"PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092\" >> \"$GITHUB_ENV\"\necho\ + \ \"PANTS_REMOTE_CACHE_READ=true\" >> \"$GITHUB_ENV\"\necho \"PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}\" >> \"$GITHUB_ENV\"\ + \n" - name: Install AdoptJDK uses: actions/setup-java@v3 with: diff --git a/build-support/bin/generate_github_workflows.py b/build-support/bin/generate_github_workflows.py index c628df96b3d..b6671e3c02f 100644 --- a/build-support/bin/generate_github_workflows.py +++ b/build-support/bin/generate_github_workflows.py @@ -186,6 +186,58 @@ def checkout(*, fetch_depth: int = 10, containerized: bool = False) -> Sequence[ return steps +def launch_bazel_remote() -> Sequence[Step]: + """Run a sidecar bazel-remote instance. + + This process proxies to a public-read/private-write S3 bucket (cache.pantsbuild.org). PRs within + pantsbuild/pants will have AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY secrets set and so will be + able to read and write the cache. PRs across forks will not, so they use hard-coded read only + creds so they can at least read from the cache. + """ + return [ + { + "name": "Launch bazel-remote", + "run": dedent( + """\ + mkdir -p ~/bazel-remote + if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then + CACHE_WRITE=false + # If no secret read/write creds, use hard-coded read-only creds, so that + # cross-fork PRs can at least read from the cache. + # These creds are hard-coded here in this public repo, which makes the bucket + # world-readable. But since putting raw AWS tokens in a public repo, even + # deliberately, is icky, we base64-them. This will at least help hide from + # automated scanners that look for checked in AWS keys. + # Not that it would be terrible if we were scanned, since this is public + # on purpose, but it's best not to draw attention. + AWS_ACCESS_KEY_ID=$(echo 'QUtJQVY2QTZHN1JRVkJJUVM1RUEK' | base64 -d) + AWS_SECRET_ACCESS_KEY=$(echo 'd3dOQ1k1eHJJWVVtejZBblV6M0l1endXV0loQWZWcW9GZlVjMDlKRwo=' | base64 -d) + else + CACHE_WRITE=true + fi + docker run --detach -u 1001:1000 \ + -v ~/bazel-remote:/data \ + -p 9092:9092 \ + buchgr/bazel-remote-cache:v2.4.1 \ + --s3.auth_method=access_key \ + --s3.access_key_id="${AWS_ACCESS_KEY_ID}" \ + --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" \ + --s3.bucket=cache.pantsbuild.org \ + --s3.endpoint=s3.us-east-1.amazonaws.com \ + --max_size 30 + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + echo "PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}" >> "$GITHUB_ENV" + """ + ), + "env": { + "AWS_ACCESS_KEY_ID": f"{gha_expr('secrets.AWS_ACCESS_KEY_ID')}", + "AWS_SECRET_ACCESS_KEY": f"{gha_expr('secrets.AWS_SECRET_ACCESS_KEY')}", + }, + } + ] + + def global_env() -> Env: return { "PANTS_CONFIG_FILES": "+['pants.ci.toml']", @@ -590,7 +642,11 @@ def bootstrap_jobs( def test_jobs( - helper: Helper, python_versions: list[str], shard: str | None, platform_specific: bool + helper: Helper, + python_versions: list[str], + shard: str | None, + platform_specific: bool, + with_remote_caching: bool, ) -> Jobs: human_readable_job_name = f"Test Python ({helper.platform_name()})" human_readable_step_name = "Run Python tests" @@ -621,6 +677,7 @@ def test_jobs( "if": IS_PANTS_OWNER, "steps": [ *checkout(), + *(launch_bazel_remote() if with_remote_caching else []), install_jdk(), *( [install_go(), download_apache_thrift()] @@ -645,7 +702,9 @@ def linux_x86_64_test_jobs(python_versions: list[str]) -> Jobs: helper = Helper(Platform.LINUX_X86_64) def test_python_linux(shard: str) -> dict[str, Any]: - return test_jobs(helper, python_versions, shard, platform_specific=False) + return test_jobs( + helper, python_versions, shard, platform_specific=False, with_remote_caching=True + ) shard_name_prefix = helper.job_name("test_python") jobs = { @@ -676,7 +735,7 @@ def linux_arm64_test_jobs(python_versions: list[str]) -> Jobs: rust_testing=RustTesting.SOME, ), helper.job_name("test_python"): test_jobs( - helper, python_versions, shard=None, platform_specific=True + helper, python_versions, shard=None, platform_specific=True, with_remote_caching=False ), } return jobs @@ -692,7 +751,7 @@ def macos11_x86_64_test_jobs(python_versions: list[str]) -> Jobs: rust_testing=RustTesting.SOME, ), helper.job_name("test_python"): test_jobs( - helper, python_versions, shard=None, platform_specific=True + helper, python_versions, shard=None, platform_specific=True, with_remote_caching=False ), } return jobs @@ -798,6 +857,7 @@ def test_workflow_jobs(python_versions: list[str], *, cron: bool) -> Jobs: "if": IS_PANTS_OWNER, "steps": [ *checkout(), + *launch_bazel_remote(), *linux_x86_64_helper.setup_primary_python(), linux_x86_64_helper.native_binaries_download(), {