diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f56ad46e4f9..2954c38ab88 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -547,6 +547,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Set up Python 3.9 uses: actions/setup-python@v4 with: @@ -632,6 +649,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -670,6 +704,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -729,6 +780,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -788,6 +856,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -847,6 +932,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -906,6 +1008,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -965,6 +1084,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1024,6 +1160,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1083,6 +1236,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1142,6 +1312,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1201,6 +1388,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: @@ -1261,6 +1465,23 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 10 + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: github.repository_owner == 'pantsbuild' + name: Launch bazel-remote + run: 'mkdir -p ~/bazel-remote + + docker run -u 1001:1000 -v ~/bazel-remote:/data -p 9092:9092 buchgr/bazel-remote-cache --s3.auth_method=access_key --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" --s3.access_key_id="${AWS_ACCESS_KEY_ID}" --s3.bucket=cache.pantsbuild.org --s3.endpoint=s3.us-east-1.amazonaws.com --max_size + 30 & + + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + + ' - name: Install AdoptJDK uses: actions/setup-java@v3 with: diff --git a/build-support/bin/generate_github_workflows.py b/build-support/bin/generate_github_workflows.py index 8dc953d6685..f3503f076c8 100644 --- a/build-support/bin/generate_github_workflows.py +++ b/build-support/bin/generate_github_workflows.py @@ -198,6 +198,38 @@ def checkout( return steps +def launch_bazel_remote() -> Sequence[Step]: + return [ + { + "name": "Launch bazel-remote", + "if": "github.repository_owner == 'pantsbuild'", + "run": dedent( + """\ + mkdir -p ~/bazel-remote + docker run -u 1001:1000 \ + -v ~/bazel-remote:/data \ + -p 9092:9092 \ + buchgr/bazel-remote-cache \ + --s3.auth_method=access_key \ + --s3.secret_access_key="${AWS_SECRET_ACCESS_KEY}" \ + --s3.access_key_id="${AWS_ACCESS_KEY_ID}" \ + --s3.bucket=cache.pantsbuild.org \ + --s3.endpoint=s3.us-east-1.amazonaws.com \ + --max_size 30 \ + & + echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" + echo "PANTS_REMOTE_CACHE_WRITE=true" >> "$GITHUB_ENV" + echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" + """ + ), + "env": { + "AWS_SECRET_ACCESS_KEY": f"{gha_expr('secrets.AWS_SECRET_ACCESS_KEY')}", + "AWS_ACCESS_KEY_ID": f"{gha_expr('secrets.AWS_ACCESS_KEY_ID')}", + }, + } + ] + + def global_env() -> Env: return { "PANTS_CONFIG_FILES": "+['pants.ci.toml']", @@ -615,6 +647,7 @@ def test_jobs(helper: Helper, shard: str | None, platform_specific: bool) -> Job "if": IS_PANTS_OWNER, "steps": [ *checkout(), + *launch_bazel_remote(), install_jdk(), *( [install_go(), download_apache_thrift()] @@ -798,6 +831,7 @@ def test_workflow_jobs() -> Jobs: "if": IS_PANTS_OWNER, "steps": [ *checkout(), + *launch_bazel_remote(), *linux_x86_64_helper.setup_primary_python(), *linux_x86_64_helper.native_binaries_download(), {