ci: remote multiplatform builder

.github/workflows/ci.yml

@@ -32,61 +32,28 @@ concurrency:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
-  # Name these so they look less similar than AMD/ARM; omit 64 as uninformative.
-  X86_PLATFORM: linux/amd64
-  ARM_PLATFORM: linux/arm64/v8
 
 jobs:
-  platforms:
-    runs-on: ubuntu-latest
-    outputs:
-      platforms: '${{ steps.platforms.outputs.platforms }}'
-    steps:
-      - name: Compute Docker platforms
-        id: platforms
-        run: |
-          if ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }}; then
-            # JSON-encoded list consisting only of the default platform.
-            platforms='["'"$X86_PLATFORM"'"]'
-          else
-            platforms='["$X86_PLATFORM","$ARM_PLATFORM"]'
-          fi
-          echo "platforms=$platforms" >> $GITHUB_OUTPUT
-
   # see https://docs.docker.com/build/ci/github-actions/test-before-push/
   test-proposals:
-    needs: [platforms]
-    # UNTIL https://github.com/Agoric/agoric-3-proposals/issues/2
-    timeout-minutes: 120
-    strategy:
-      matrix:
-        platform: ${{ fromJSON(needs.platforms.outputs.platforms) }}
-    # Run on our own self-hosted ARM64 machine if the platform is ARMish.
-    runs-on: ${{ contains(matrix.platform, '/arm') && fromJSON('["self-hosted","Linux","ARM64"]') || 'ubuntu-latest' }}
+    permissions:
+      # allow issuing OIDC tokens for this workflow run
+      id-token: write
+      # allow at least reading the repo contents, add other permissions if necessary
+      contents: read
+      # to push the resulting images
+      packages: write
+    runs-on: 'ubuntu-latest'
     steps:
-      - name: free up disk space
-        if: ${{ !contains(matrix.platform, '/arm') }}
-        run: |
-          # Workaround to provide additional free space for testing.
-          #   https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
-          # If this turns out not to be enough, maybe look instead at
-          #   https://github.com/actions/runner-images/issues/2840#issuecomment-1540506686
-          df -h
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /opt/ghc
-          sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-          echo "=== After cleanup:"
-          df -h
-
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - uses: depot/setup-action@v1
+        with:
+          oidc: true # to set DEPOT_TOKEN for later steps
 
-      - name: Set up QEMU for cross-platform builds
-        uses: docker/setup-qemu-action@v3
+      # make Docker's CLI use depot to build
+      - run: depot configure-docker
 
       - name: Log in to the Container registry
         uses: docker/login-action@v3
@@ -102,49 +69,21 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      - name: Compute Docker tags
-        id: docker-tags
-        run: |
-          sep=
-          # A list of comma-separated tags to merge in the final image.
-          SUFFIXED=
-          # Our platform, replacing slashes with underscores.
-          uarch=$(echo "${{ matrix.platform }}" | tr / _)
-          for TAG in ${{ steps.meta.outputs.tags }}; do
-            SUFFIXED="$SUFFIXED$sep$TAG-$uarch"
-            if test -z "$sep"; then
-              # The first tag (suffixed with our architecture) is the one we build.
-              sep=,
-              echo "tag=$TAG-$uarch" | tee -a $GITHUB_OUTPUT
-            fi
-          done
-          echo "tags=$SUFFIXED" | tee -a $GITHUB_OUTPUT
-
       # The .ts scripts depend upon this
       - run: tsx --version || npm install --global tsx
       # Enable corepack for packageManager config
       - run: corepack enable || sudo corepack enable
       - run: yarn install
 
-      - run: docker system df
-      - run: docker buildx du --verbose
-      - run: df -h
+      - name: Build proposal "use" images
+        run: yarn build
 
       # Test before pushing the images.
       - name: Build and run proposal tests
-        if: ${{ matrix.platform == env.X86_PLATFORM }}
         run: yarn test
 
-      - run: docker system df
-      - run: docker buildx du --verbose
-      - run: df -h
-
-      # Build a "use" image for each proposal. This uses Docker Bake's
-      # matrix feature. We could have each "use" image built in a different runner
-      # by including https://github.com/docker/bake-action?tab=readme-ov-file#list-targets
-      # in the GHA matrix, but that wouldn't be able to resolve the DAG of what to build first.
       - name: Push proposal "use" images
-        uses: docker/bake-action@v4
+        uses: depot/bake-action@v1
         # If we pushed from PRs, each one would overwrite main's (e.g. use-upgrade-8)
         # To push PR "use" images we'll need to qualify the tag (e.g. use-upgrade-8-pr-2).
         if: ${{ github.event_name != 'pull_request' }}
@@ -154,81 +93,4 @@ jobs:
             ./docker-bake.hcl
             ${{ steps.meta.outputs.bake-file }}
           targets: use
-
-      - run: docker system df
-      - run: docker buildx du --verbose
-      - run: df -h
-
-      - name: Build and push default image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          platforms: ${{ matrix.platform }}
-          # push to registry on every repo push. A PR #2 will push with tag `pr-2` and `main` will have tag `main`.
-          # See https://github.com/docker/metadata-action?tab=readme-ov-file#basic.
           push: true
-          tags: ${{ steps.docker-tags.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-
-      - run: docker system df
-      - run: docker buildx du --verbose
-      - run: df -h
-
-  # Merge the default image from each platform into one multi-arch image,
-  # then publish that multiarch image.
-  docker-publish-multiarch:
-    needs: [test-proposals, platforms]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          buildkitd-flags: --debug
-
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          registry: ${{ env.REGISTRY }}
-      - name: Compute tags
-        id: docker-tags
-        run: |
-          echo "tags=${{ steps.meta.outputs.tags }}" >> $GITHUB_OUTPUT
-
-      - name: Push multiarch image
-        run: |
-          set -ex
-          # Push all tags, comprised of all architectures, to the registry.
-          for TAG in ${{ steps.docker-tags.outputs.tags }}; do
-            sources=
-            for ARCH in ${{ join(fromJson(needs.platforms.outputs.platforms), ' ') }}; do
-              uarch=$(echo "$ARCH" | tr / _)
-              BUILD_TAG="$TAG-$uarch"
-              sources="$sources $BUILD_TAG"
-            done
-            docker buildx imagetools create --tag "$TAG"$sources
-          done
-
-      - name: notify on failure
-        if: failure() && github.event_name != 'pull_request'
-        uses: ./.github/actions/notify-status
-        with:
-          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
-          from: ${{ secrets.NOTIFY_EMAIL_FROM }}
-          to: ${{ secrets.NOTIFY_EMAIL_TO }}
-          password: ${{ secrets.NOTIFY_EMAIL_PASSWORD }}

depot.json

@@ -0,0 +1 @@
+{"id":"bqgtmlhmh8"}

package.json

@@ -12,7 +12,11 @@
     "@agoric/synthetic-chain": "workspace:*"
   },
   "agoricSyntheticChain": {
-    "fromTag": null
+    "fromTag": null,
+    "platforms": [
+      "linux/amd64",
+      "linux/arm64"
+    ]
   },
   "license": "Apache-2.0",
   "packageManager": "yarn@4.0.2",

packages/synthetic-chain/cli.ts

@@ -58,14 +58,14 @@ const prepareDockerBuild = () => {
   execSync(`cp -r ${path.resolve(cliPath, '..', 'upgrade-test-scripts')} .`);
   execSync(`cp -r ${path.resolve(cliPath, '..', 'docker-bake.hcl')} .`);
   writeDockerfile(allProposals, buildConfig.fromTag);
-  writeBakefileProposals(allProposals);
+  writeBakefileProposals(allProposals, buildConfig.platforms);
   buildProposalSubmissions(proposals);
 };
 
 switch (cmd) {
   case 'build': {
     prepareDockerBuild();
-    bakeTarget('use', values.dry);
+    bakeTarget('use', values.dry, buildConfig.platforms);
     break;
   }
   case 'test':

packages/synthetic-chain/docker-bake.hcl

@@ -18,6 +18,7 @@ group "default" {
 target "use" {
   inherits = ["docker-metadata-action"]
   name = "use-${proposal}"
+  platforms = PLATFORMS
   matrix = {
     proposal = PASSED_PROPOSALS
   }

packages/synthetic-chain/src/cli/build.ts

@@ -3,13 +3,16 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { ProposalInfo } from './proposals.js';
 
+export type Platform = 'linux/amd64' | 'linux/arm64';
+
 export type AgoricSyntheticChainConfig = {
   /**
    * The agoric-3-proposals tag to build the agoric synthetic chain from.
    * If `null`, the chain is built from an ag0 genesis.
    * Defaults to `main`, which containing all passed proposals
    */
   fromTag: string | null;
+  platforms?: Platform[];
 };
 
 const defaultConfig: AgoricSyntheticChainConfig = {
@@ -64,16 +67,25 @@ export const buildProposalSubmissions = (proposals: ProposalInfo[]) => {
 
 /**
  * Bake images using the docker buildx bake command.
- * 
+ *
  * Note this uses `--load` which pushes the completed images to the builder,
  * consuming 2-3 GB per image.
  * @see {@link https://docs.docker.com/engine/reference/commandline/buildx_build/#load}
  *
  * @param target - The image or group target
  * @param [dry] - Whether to skip building and just print the build config.
  */
-export const bakeTarget = (target: string, dry = false) => {
-  const cmd = `docker buildx bake --load ${target} ${dry ? '--print' : ''}`;
+export const bakeTarget = (
+  target: string,
+  dry = false,
+  platforms?: Platform[],
+) => {
+  const cmd = platforms
+    ? // Docker buildx does not support multiplatform in one builder so use Depot remote builder
+      // Without this: ERROR: docker exporter does not currently support exporting manifest lists
+      // See https://github.com/docker/roadmap/issues/371
+      `depot bake --load ${target} ${dry ? '--print' : ''}`
+    : `docker buildx bake --load ${target} ${dry ? '--print' : ''}`;
   console.log(cmd);
   execSync(cmd, { stdio: 'inherit' });
 };

packages/synthetic-chain/src/cli/dockerfileGen.ts

@@ -10,6 +10,7 @@ import {
   imageNameForProposal,
   isPassed,
 } from './proposals.js';
+import { Platform } from './build.ts';
 
 /**
  * Templates for Dockerfile stages
@@ -196,6 +197,9 @@ export function writeBakefileProposals(
 
   const json = {
     variable: {
+      PLATFORMS: {
+        default: platforms || null,
+      },
       ALL_PROPOSALS: {
         default: allProposals.map(p => p.proposalName),
       },