feat: use docker in frontend GHA to parallelize work

.dockerignore

@@ -34,7 +34,6 @@
 **/*.sqllite
 **/*.swp
 **/.terser-plugin-cache/
-**/.storybook/
 **/node_modules/
 
 tests/

.github/workflows/superset-docs-verify.yml

@@ -24,7 +24,7 @@ jobs:
       - uses: JustinBeckwith/linkinator-action@v1.11.0
         continue-on-error: true # This will make the job advisory (non-blocking, no red X)
         with:
-          paths: "**/*.md, **/*.mdx"
+          paths: "**/*.md, **/*.mdx, !superset-frontend/CHANGELOG.md"
           linksToSkip: >-
             ^https://github.com/apache/(superset|incubator-superset)/(pull|issue)/\d+,
             http://localhost:8088/,

.github/workflows/superset-frontend.yml

@@ -1,4 +1,4 @@
-name: Frontend
+name: "Frontend Build CI (unit tests, linting & sanity checks)"
 
 on:
   push:
@@ -21,60 +21,166 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          submodules: recursive
-      - name: Check npm lock file version
-        run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json
+
       - name: Check for file changes
         id: check
         uses: ./.github/actions/change-detector/
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Setup Node.js
-        if: steps.check.outputs.frontend
-        uses: actions/setup-node@v4
-        with:
-          node-version: "20"
-      - name: Install dependencies
+
+      - name: Setup Docker Environment
         if: steps.check.outputs.frontend
-        uses: ./.github/actions/cached-dependencies
+        uses: ./.github/actions/setup-docker
         with:
-          run: npm-install
-      - name: eslint
+          dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
+          dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
+          build: "true"
+
+      - name: Setup supersetbot
         if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
-        run: |
-          npm run eslint -- . --quiet
-      - name: tsc
+        uses: ./.github/actions/setup-supersetbot/
+
+      - name: Build Docker Image
         if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          npm run type
-      - name: Build plugins packages
-        if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
-        run: npm run plugins:build
-      - name: Build plugins Storybook
-        if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
-        run: npm run plugins:build-storybook
-      - name: superset-ui/core coverage
+          TAG="superset-node-${{ github.sha }}"
+
+          supersetbot docker \
+            --load \
+            --preset superset-node-ci \
+            --platform "linux/amd64" \
+            --extra-flags "--tag $TAG"
+
+          docker save $TAG | gzip > superset-node.tar.gz
+
+      - name: Upload Docker Image Artifact
         if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-image
+          path: superset-node.tar.gz
+
+  sharded-jest-tests:
+    needs: frontend-build
+    if: needs.frontend-build.result == 'success'
+    strategy:
+      matrix:
+        shard: [1, 2, 3, 4, 5, 6, 7, 8]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Download Docker Image Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker Image
         run: |
-          npm run core:cover
-      - name: unit tests
-        if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
+          docker load -i superset-node.tar.gz
+
+      - name: npm run test with coverage
         run: |
-          npm run test -- --coverage --silent
-      # todo: remove this step when fix generator as a project in root jest.config.js
-      - name: generator-superset unit tests
-        if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend/packages/generator-superset
-        run: npm run test
+          docker run \
+          --rm superset-node-${{ github.sha }} \
+          -v ${{ github.workspace }}/superset-frontend:/app/superset-frontend \
+          bash -c \
+          'npm run test -- --coverage --shard=${{ matrix.shard }}/8 --coverageReporters=json-summary'
+          find ${{ github.workspace }}/superset-frontend/coverage
+
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-artifacts-${{ matrix.shard }}
+          path: coverage/
+
+  report-coverage:
+    needs: [sharded-jest-tests]
+    if: needs.frontend-build.result == 'success'
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Download Docker Image Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: coverage-artifacts
+          merge-multiple: true
+          pattern: coverage/*
+
+      - name: Show files
+        run: find .
+
+      - name: Merge Code Coverage
+        run: npx nyc merge coverage/ merged-output/merged-coverage.json
+
       - name: Upload code coverage
         uses: codecov/codecov-action@v5
         with:
           flags: javascript
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
+
+  core-cover:
+    needs: frontend-build
+    if: needs.frontend-build.result == 'success'
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Download Docker Image Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker Image
+        run: |
+          docker load -i superset-node.tar.gz
+
+      - name: superset-ui/core coverage
+        run: |
+          docker run --rm superset-node-${{ github.sha }} bash -c \
+          "npm run core:cover"
+
+  lint-frontend:
+    needs: frontend-build
+    if: needs.frontend-build.result == 'success'
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Download Docker Image Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker Image
+        run: |
+          docker load -i superset-node.tar.gz
+
+      - name: eslint
+        run: |
+          docker run --rm superset-node-${{ github.sha }} bash -c \
+          "npm i && npm run eslint -- . --quiet"
+      - name: tsc
+        run: |
+          docker run --rm superset-node-${{ github.sha }} bash -c \
+          "npm run type"
+
+  validate-frontend:
+    needs: frontend-build
+    if: needs.frontend-build.result == 'success'
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Download Docker Image Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker Image
+        run: |
+          docker load -i superset-node.tar.gz
+
+      - name: Build plugins packages
+        run: |
+          docker run --rm superset-node-${{ github.sha }} bash -c \
+          "npm run plugins:build"
+      - name: Build plugins Storybook
+        run: |
+          docker run --rm superset-node-${{ github.sha }} bash -c \
+          "npm run plugins:build-storybook"

Dockerfile

@@ -24,9 +24,9 @@ ARG PY_VER=3.10-slim-bookworm
 ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
 
 ######################################################################
-# superset-node used for building frontend assets
+# superset-node-ci used as a base for building frontend assets and CI
 ######################################################################
-FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
+FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node-ci
 ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
 ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
 ARG DEV_MODE="false"           # Skip frontend build in dev mode
@@ -53,6 +53,10 @@ RUN mkdir -p /app/superset/static/assets \
              /app/superset/translations
 
 # Mount package files and install dependencies if not in dev mode
+# NOTE: we mount packages and plugins as they are referenced in package.json as workspaces
+# ideally we'd COPY only their package.json. Here npm ci will be cached as long
+# as the full content of these folders don't change, yielding a decent cache reuse rate.
+# Note that's it's not possible selectively COPY of mount using blobs.
 RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \
     --mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \
     --mount=type=cache,target=/root/.cache \
@@ -66,6 +70,11 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j
 # Runs the webpack build process
 COPY superset-frontend /app/superset-frontend
 
+######################################################################
+# superset-node used for compile frontend assets
+######################################################################
+FROM superset-node-ci AS superset-node
+
 # Build the frontend if not in dev mode
 RUN --mount=type=cache,target=/app/superset-frontend/.temp_cache \
     --mount=type=cache,target=/root/.npm \