chore(ci): Convert CI workflows to reusable & apply various QoL chang…

…es (grain-lang#1770)

* chore(ci): Update checkout action to v3

* chore(compiler): Add shx and postimport-dependencies cleanup script

* chore(ci): Convert CI workflows to reusable & apply various QoL changes

* Apply suggestions from code review

* chore(ci): Add a concurrency group to our CI workflow

* empty commit to show concurrency group

.github/workflows/build-js.yml

@@ -0,0 +1,115 @@
+# This workflow exists to provide a way to dispatch a CI run for any
+# given ref on any of our OS targets. It can also be consumed in our
+# various other builds.
+name: (js) Build and test
+
+on:
+  workflow_dispatch:
+    inputs:
+      os:
+        description: "Operating system to run CI on"
+        required: true
+        default: "ubuntu-latest"
+        type: choice
+        options:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+  workflow_call:
+    inputs:
+      os:
+        type: string
+        required: true
+
+jobs:
+  build:
+    name: (js) Build and test
+    runs-on: ${{ inputs.os }}
+
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v3
+
+      - name: Setup node.js
+        uses: actions/setup-node@v3.6.0
+        with:
+          node-version: "16"
+          check-latest: true
+          cache: "npm"
+
+      - name: Install npm dependencies
+        run: |
+          npm ci
+
+      - name: Esy cache
+        id: esy-cache
+        uses: actions/cache/restore@v3
+        with:
+          path: compiler/_export
+          key: ${{ runner.os }}-esy-${{ hashFiles('compiler/esy.lock/index.json') }}
+
+      - name: Esy setup
+        # Don't crash the run if esy cache import fails - mostly happens on Windows
+        continue-on-error: true
+        run: |
+          npm run compiler prepare
+          npm run compiler import-dependencies
+
+      # Don't build native executables, only the JS builds
+      - name: Build compiler
+        run: |
+          npm run compiler build:js
+
+      - name: Run tests
+        run: |
+          npm run compiler test:js
+
+      # This will log a warning because we didn't build the native exe files
+      - name: Build pkg binary for Windows
+        if: inputs.os == 'ubuntu-latest'
+        run: |
+          npm run cli build-pkg -- --target=win-x64
+          tar -cvf grain.tar -C pkg grain.exe
+
+      - name: Upload pkg binary for Windows
+        if: inputs.os == 'ubuntu-latest'
+        uses: actions/upload-artifact@v3
+        with:
+          path: ./grain.tar
+          name: grain-win-x64
+
+      # This will log a warning because we didn't build the native exe files
+      - name: Build pkg binary for Mac
+        if: inputs.os == 'ubuntu-latest'
+        run: |
+          npm run cli build-pkg -- --target=mac-x64
+          tar -cvf grain.tar -C pkg grain
+
+      - name: Upload pkg binary for Mac
+        if: inputs.os == 'ubuntu-latest'
+        uses: actions/upload-artifact@v3
+        with:
+          path: ./grain.tar
+          name: grain-mac-x64
+
+      # This will log a warning because we didn't build the native exe files
+      - name: Build pkg binary for Linux
+        if: inputs.os == 'ubuntu-latest'
+        run: |
+          npm run cli build-pkg -- --target=linux-x64
+          tar -cvf grain.tar -C pkg grain
+
+      - name: Upload pkg binary for Linux
+        if: inputs.os == 'ubuntu-latest'
+        uses: actions/upload-artifact@v3
+        with:
+          path: ./grain.tar
+          name: grain-linux-x64
+
+      # This is to test that we didn't actually introduce multivalue
+      # which might happen through a binaryen optimization
+      - name: Run NEAR smoketest
+        if: inputs.os != 'windows-latest'
+        run: |
+          npm run stdlib clean
+          npm run cli test

.github/workflows/build-native.yml

@@ -0,0 +1,124 @@
+# This workflow exists to provide a way to dispatch a CI run for any
+# given ref on any of our OS targets. It can also be consumed in our
+# various other builds.
+name: (native) Build and test
+
+on:
+  workflow_dispatch:
+    inputs:
+      os:
+        description: "Operating system to run CI on"
+        required: true
+        default: "ubuntu-latest"
+        type: choice
+        options:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+  workflow_call:
+    inputs:
+      os:
+        type: string
+        required: true
+
+jobs:
+  build:
+    name: (native) Build and test
+    runs-on: ${{ inputs.os }}
+
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v3
+
+      - name: Setup node.js
+        uses: actions/setup-node@v3.6.0
+        with:
+          node-version: "16"
+          check-latest: true
+          cache: "npm"
+
+      - name: Install npm dependencies
+        run: |
+          npm ci
+
+      - name: Esy cache
+        id: esy-cache
+        uses: actions/cache/restore@v3
+        with:
+          path: compiler/_export
+          key: ${{ runner.os }}-esy-${{ hashFiles('compiler/esy.lock/index.json') }}
+
+      - name: Esy setup
+        # Don't crash the run if esy cache import fails - mostly happens on Windows
+        continue-on-error: true
+        run: |
+          npm run compiler prepare
+          npm run compiler import-dependencies
+
+      - name: Build compiler
+        run: |
+          npm run compiler build
+
+      # Upload the artifacts before we run the tests so we
+      # can download to debug if tests fail in a weird way
+      - name: Upload native compiler artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: native-build-artifacts-${{ runner.os }}-${{ runner.arch }}
+          path: cli/bin/*.exe
+
+      - name: Run tests
+        run: |
+          npm run compiler test
+
+      - name: (compiler) Check parser error messages exhaustiveness
+        run: |
+          npm run compiler parser:check-errors
+
+      - name: (graindoc) Check parser error messages exhaustiveness
+        run: |
+          npm run compiler graindoc-parser:check-errors
+
+      # Check formatting last because building is more important
+      - name: (compiler) Check formatting
+        if: inputs.os != 'windows-latest'
+        run: |
+          npm run compiler check-format
+
+      - name: (js-runner) Check formatting
+        if: inputs.os != 'windows-latest'
+        run: |
+          npm run js-runner check-format
+
+      - name: (cli) Check formatting
+        if: inputs.os != 'windows-latest'
+        run: |
+          npm run cli check-format
+
+      # This is to test the CLI is working
+      - name: Log Grain version
+        run: |
+          grain -v
+
+      # If we have a working grain CLI, we can run graindoc on stdlib
+      - name: (stdlib) Check documentation
+        if: inputs.os != 'windows-latest'
+        run: |
+          grain doc stdlib -o stdlib --current-version=$(grain -v)
+          git diff --exit-code --name-only
+
+      # If we have a working grain CLI, we can run grainfmt on stdlib & tests
+      - name: (stdlib) Check formatting
+        if: inputs.os != 'windows-latest'
+        run: |
+          grain format stdlib -o stdlib
+          grain format compiler/test/stdlib -o compiler/test/stdlib
+          git diff --exit-code --name-only
+
+      # This is to test that we didn't actually introduce multivalue
+      # which might happen through a binaryen optimization
+      - name: Run NEAR smoketest
+        if: inputs.os != 'windows-latest'
+        run: |
+          npm run stdlib clean
+          npm run cli test

.github/workflows/build.yml

@@ -0,0 +1,34 @@
+# This workflow exists so a dependency cache can be built for a
+# single OS and then its native & js builds can be run in parallel.
+#
+# If we instead built the dependency cache with a matrix in `ci.yml`,
+# it'd block every build until all caches were built for all our targets.
+name: Build and test
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+        required: true
+
+jobs:
+  dependencies:
+    name: Dependencies
+    uses: ./.github/workflows/dependencies.yml
+    with:
+      os: ${{ inputs.os }}
+
+  build-native:
+    name: (native) Build and test
+    needs: [dependencies]
+    uses: ./.github/workflows/build-native.yml
+    with:
+      os: ${{ inputs.os }}
+
+  build-js:
+    name: (js) Build and test
+    needs: [dependencies]
+    uses: ./.github/workflows/build-js.yml
+    with:
+      os: ${{ inputs.os }}

.github/workflows/ci.yml

@@ -1,106 +1,24 @@
 name: Grain CI Workflow
+
 on: [push, pull_request]
 
+# This will cancel previous runs when a branch or PR is updated
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   build:
-    name: Build and test
-    runs-on: ${{ matrix.os }}
-
+    name: Build and test on ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        node-version: ["16"]
-
-    steps:
-      - name: Checkout project
-        uses: actions/checkout@v2
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v3.1.1
-        with:
-          node-version: ${{ matrix.node-version }}
-          check-latest: true
-          cache: "npm"
-
-      # Adds `shx` globally for cross-platform shell commands
-      - name: Setup environment (All)
-        run: |
-          npm install -g shx
-
-      - name: Set up JS runner and CLI
-        run: |
-          npm ci
-
-      - name: Esy setup
-        run: |
-          npm run compiler prepare
-
-      - name: Esy cache
-        id: esy-cache
-        uses: actions/cache@v2
-        with:
-          path: compiler/_export
-          key: ${{ runner.os }}-esy-${{ hashFiles('compiler/esy.lock/index.json') }}
-
-      - name: Import esy cache
-        if: steps.esy-cache.outputs.cache-hit == 'true'
-        # Don't crash the run if esy cache import fails - mostly happens on Windows
-        continue-on-error: true
-        run: |
-          npm run compiler import-dependencies
-          shx rm -rf compiler/_export
-
-      - name: Build compiler
-        run: |
-          npm run compiler build
-
-      # Re-export dependencies if anything has changed or if it is the first time
-      - name: Build esy cache
-        if: steps.esy-cache.outputs.cache-hit != 'true'
-        run: |
-          npm run compiler export-dependencies
-
-      - name: Run tests (native)
-        run: |
-          npm run compiler test
-
-      # Windows still needs some fixes to run correctly in JS
-      - name: Run tests (js)
-        run: |
-          npm run compiler test:js
-
-      - name: Check parser error messages exhaustiveness
-        run: |
-          npm run compiler parser:check-errors
-          npm run compiler graindoc-parser:check-errors
-
-      # Formatting lint last because building is more important
-      - name: Run formatting lint
-        if: matrix.os != 'windows-latest'
-        run: |
-          npm run compiler check-format
-          npm run js-runner check-format
-          npm run cli check-format
-
-      # This is to test the CLI is working
-      - name: Log Grain version
-        run: |
-          grain -v
-
-      # If we have a working grain CLI, we can run grainfmt and graindoc on stdlib
-      - name: Check stdlib docs and formatting
-        if: matrix.os != 'windows-latest'
-        run: |
-          grain doc stdlib -o stdlib --current-version=$(grain -v)
-          grain format stdlib -o stdlib
-          grain format compiler/test/stdlib -o compiler/test/stdlib
-          git diff --exit-code --name-only
-
-      # This is to test that we didn't actually introduce multivalue
-      # which might happen through a binaryen optimization
-      - name: Run NEAR smoketest
-        if: matrix.os != 'windows-latest'
-        run: |
-          npm run stdlib clean
-          npm run cli test
+    uses: ./.github/workflows/build.yml
+    with:
+      os: ${{ matrix.os }}
+
+  test-pkg:
+    name: Test pkg binaries
+    needs: [build]
+    uses: ./.github/workflows/test-pkg.yml