Improvements for CI in GitHub Actions

.github/workflows/_combine_coverage_data.yml

@@ -0,0 +1,163 @@
+# This workflow will download simplecov artifacts created by parallel_test and combine
+# them into one set of test coverage data.
+#
+# This workflow is meant to be called by other workflows.
+name: "♻️  SimpleCov Report"
+on:
+  workflow_call:
+
+jobs:
+  combine_coverage_data:
+    name: "♻️ "
+    runs-on: ubuntu-latest
+    env:
+      RAILS_ENV: test
+      # Manually export your local RAILS_MASTER_KEY if using the credentials system.
+      # RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }}
+      BUNDLE_JOBS: 2
+      BUNDLE_RETRY: 3
+    steps:
+      - name: Checkout Starter Repo
+        uses: bullet-train-co/checkout-repo-with-matching-branch@v1
+        with:
+          target_dir: tmp/starter
+          repository: bullet-train-co/bullet_train
+
+      # We have to put the core code into place so that simplecov will recognize
+      # it as being part of the probject when it collates all the coverage data.
+      - name: Checkout Core Repo
+        uses: actions/checkout@v4
+        with:
+          path: tmp/starter/local/bullet_train-core
+
+      - name: Install Ruby and gems
+        uses: ruby/setup-ruby@v1
+        with:
+          working-directory: tmp/starter
+          bundler-cache: true
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: tmp/starter/coverage_artifacts
+          pattern: test_coverage_*.log
+          merge-multiple: false
+
+      - name: List downloaded artifacts
+        run: ls -alR tmp/starter/coverage_artifacts
+        shell: bash
+
+      - name: Cat an artifact
+        run: cat tmp/starter/coverage_artifacts/test_coverage_*.log/.resultset.json
+        shell: bash
+
+      - name: Combine Coverage Data
+        working-directory: tmp/starter
+        run: "bundle exec rake coverage:report[coverage_artifacts/**/.resultset.json]"
+        shell: bash
+
+      - name: Copy coverage.json to coverage_with_stats.json
+        run: cp tmp/starter/coverage/coverage.json tmp/starter/coverage/coverage_with_stats.json
+        continue-on-error: true
+        shell: bash
+
+      - name: Combine Coverage Data With Groups
+        working-directory: tmp/starter
+        run: "bundle exec rake coverage:report_with_gem_groups[coverage_artifacts/**/.resultset.json]"
+        shell: bash
+
+      - name: Copy coverage.json to coverage_with_groups.json
+        run: cp tmp/starter/coverage/coverage.json tmp/starter/coverage/coverage_with_groups.json
+        continue-on-error: true
+        shell: bash
+
+      - name: List coverage dir
+        run: ls -alR tmp/starter/coverage
+        continue-on-error: true
+        shell: bash
+
+      - name: Upload Coverage Report
+        uses: actions/upload-artifact@v4
+        id: upload-coverage-report
+        with:
+          name: test_coverage_report
+          path: tmp/starter/coverage
+          include-hidden-files: true
+          retention-days: 1
+
+      - uses: actions/github-script@v7
+        id: create-summary
+        name: Summary
+        env:
+          COVERAGE_REPORT_LINK: ${{ steps.upload-coverage-report.outputs.artifact-url }}
+        with:
+          script: |
+            try {
+              const fs = require('fs')
+              const jsonString = fs.readFileSync('tmp/starter/coverage/coverage_with_stats.json')
+              var coverageData = JSON.parse(jsonString)
+
+              const groupJsonString = fs.readFileSync('tmp/starter/coverage/coverage_with_groups.json')
+              var groupCoverageData = JSON.parse(groupJsonString)
+
+              var files = coverageData.files.sort((a, b) => a.covered_percent - b.covered_percent);
+              var numberOfFilesToShow = 10
+              files = files.slice(0,numberOfFilesToShow)
+
+              var fileDetails = []
+              fileDetails.push(`<details>`)
+              fileDetails.push(`<summary>File Coverage Details</summary>`)
+              fileDetails.push(` `)
+              fileDetails.push(`${numberOfFilesToShow} files with the least coverage`)
+              fileDetails.push(` `)
+              fileDetails.push(`| File | Coverage |`)
+              fileDetails.push(`| ---- | -------- |`)
+              for (const file of files) {
+                var filePath = file.filename.split("tmp/starter/")[1];
+                var fileCoverage = file.covered_percent.toFixed(1)
+                fileDetails.push(`| ${filePath} | ${fileCoverage}% |`)
+              }
+              fileDetails.push(` `)
+              fileDetails.push(`</details>`)
+
+              var groups = groupCoverageData.groups;
+              var groupNames = Object.keys(groups)
+
+              var groupDetails = []
+              groupDetails.push(`<details>`)
+              groupDetails.push(`<summary>Group Coverage Details</summary>`)
+              groupDetails.push(` `)
+              groupDetails.push(`| Group | Coverage |`)
+              groupDetails.push(`| ---- | -------- |`)
+              for (const groupName of groupNames) {
+                var groupCoverage = groups[groupName].lines.covered_percent.toFixed(1)
+                groupDetails.push(`| ${groupName} | ${groupCoverage}% |`)
+              }
+              groupDetails.push(` `)
+              groupDetails.push(`</details>`)
+
+              core.summary.addRaw(`${coverageData.metrics.covered_percent.toFixed(1)}% Covered`,true)
+              core.summary.addRaw(`${coverageData.files.length} Files`,true)
+              core.summary.addRaw(`${coverageData.metrics.covered_strength.toFixed(1)} Coverage Strength`,true)
+              core.summary.addRaw(`${coverageData.metrics.covered_lines} Lines Covered (of ${coverageData.metrics.total_lines})`,true)
+              core.summary.addRaw(` `,true)
+
+              core.summary.addRaw(fileDetails.join("\r\n"), true)
+              core.summary.addRaw(` `,true)
+              core.summary.addRaw(groupDetails.join("\r\n"), true)
+              core.summary.addRaw(` `,true)
+
+              core.summary.addLink('For the next 24 hours you can download a full coverage report.', process.env.COVERAGE_REPORT_LINK)
+            } catch(err) {
+              core.summary.addRaw("Error while reading or parsing the coverage JSON", true)
+              core.summary.addRaw(err.toString(), true)
+              core.error("Error while reading or parsing the coverage JSON")
+              core.setFailed(err)
+            }
+            core.summary.write()
+
+      - name: Delete artifacts
+        uses: geekyeggo/delete-artifact@v5
+        with:
+          name: test_coverage_*.log
+          failOnError: false

.github/workflows/_combine_runtime_logs.yml

@@ -0,0 +1,72 @@
+# This workflow will download runtime log artifacts created by parallel_test and combine
+# them into one log that is then used by subsequent test runs to help split up test files
+# more efficiently into groups.
+#
+# This workflow is meant to be called by other workflows.
+name: "🪵 Combine Runtime Logs for Parallel Tests"
+on:
+  workflow_call:
+
+jobs:
+  combine_runtime_logs:
+    name: "🪵"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Starter Repo
+        uses: bullet-train-co/checkout-repo-with-matching-branch@v1
+        with:
+          target_dir: tmp/starter
+          repository: bullet-train-co/bullet_train
+
+      - name: runtime log cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            tmp/starter/tmp/parallel_runtime_test.log
+          key: runtime-cache-${{ runner.os }}-${{ github.ref }}-${{ github.sha }}
+          restore-keys: |
+            runtime-cache-${{ runner.os }}-${{ github.ref }}-${{ github.sha }}
+            runtime-cache-${{ runner.os }}-${{ github.ref }}-
+            runtime-cache-${{ runner.os }}-
+
+      - name: ls tmp/starter/tmp
+        run: ls -al tmp/starter/tmp
+        shell: bash
+
+      - name: Touch Parallel Tests Runtime Log
+        run: touch tmp/parallel_runtime_test.log
+        working-directory: tmp/starter
+        shell: bash
+
+      - name: cat runtime log
+        run: cat tmp/starter/tmp/parallel_runtime_test.log
+        shell: bash
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          pattern: parallel_runtime_test_*.log
+          merge-multiple: false
+
+      - name: List downloaded artifacts
+        run: ls -R artifacts
+        shell: bash
+
+      - name: Combine Runtime Logs
+        run: cat artifacts/**/parallel_runtime_test.log > tmp/starter/tmp/parallel_runtime_test.log
+        shell: bash
+
+      - name: Cat the combined log
+        run: cat tmp/starter/tmp/parallel_runtime_test.log
+        shell: bash
+
+      - name: Line count the combined log
+        run: wc -l tmp/starter/tmp/parallel_runtime_test.log
+        shell: bash
+
+      - name: Delete artifacts
+        uses: geekyeggo/delete-artifact@v5
+        with:
+          name: parallel_runtime_test_*.log
+          failOnError: false

.github/workflows/_combine_summary_logs.yml

@@ -0,0 +1,45 @@
+# This workflow will download test summary artifacts created by parallel_test and combine
+# them into one summary report about test pass/fail status.
+#
+# This workflow is meant to be called by other workflows.
+name: "📊 Test Results"
+on:
+  workflow_call:
+
+jobs:
+  combine_summary_logs:
+    name: "📊"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          path: tmp/starter
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          pattern: test_summary_*.log
+          merge-multiple: false
+
+      - name: List downloaded artifacts
+        run: ls -R artifacts
+        shell: bash
+
+      - name: Count Test Result Files
+        run: ls -alR artifacts | grep TEST | wc -l
+        shell: bash
+
+      - name: Test Summary
+        uses: test-summary/action@v2
+        with:
+          paths: "artifacts/**/TEST-*.xml"
+        if: always()
+
+      - name: Delete artifacts
+        uses: geekyeggo/delete-artifact@v5
+        with:
+          name: test_summary_*.log
+          failOnError: false

.github/workflows/_run_super_scaffolding_tests.yml

@@ -7,9 +7,19 @@ on:
   workflow_dispatch:
 
 jobs:
+  # NOTE: This is here just to make the workflow visualization layout better.
+  # Without it the layout is really bad and confusing.
+  calculate_matrix:
+    name: 🤷 Fake Matrix
+    runs-on: ubuntu-latest
+    steps:
+      - name: Do Nothing
+        run: echo "This is a useless step that just helps things look nicer..."
+        shell: bash
   test:
     name: "🏗️"
     runs-on: ubuntu-latest
+    needs: calculate_matrix
     strategy:
       fail-fast: false
       matrix:
@@ -46,17 +56,17 @@ jobs:
       BUNDLE_JOBS: 2
       BUNDLE_RETRY: 3
     steps:
-      - name: Checkout Core Repo
-        uses: actions/checkout@v4
-        with:
-          path: tmp/core
-
       - name: Checkout Starter Repo
         uses: bullet-train-co/checkout-repo-with-matching-branch@v1
         with:
           target_dir: tmp/starter
           repository: bullet-train-co/bullet_train
 
+      - name: Checkout Core Repo
+        uses: actions/checkout@v4
+        with:
+          path: tmp/starter/local/bullet_train-core
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
@@ -77,7 +87,7 @@ jobs:
         uses: bullet-train-co/link-core-gems@v1
         with:
           application_dir: tmp/starter
-          core_dir: tmp/core
+          core_dir: tmp/starter/local/bullet_train-core
 
       # TODO: This might be a bad idea. Maybe we should just have spring in the Gemfile all the time.
       - name: Allow adding of spring
@@ -102,8 +112,17 @@ jobs:
         run: bin/rails test test/controllers/webhooks/incoming/some_provider_webhooks_controller_test.rb
         if: ${{ strategy.job-index == 5 }}
 
-      - name: Test Summary
-        uses: test-summary/action@v2
+      - name: Upload Test Summary Logs
+        uses: actions/upload-artifact@v4
+        if: always()
         with:
-          paths: "tmp/starter/test/reports/**/TEST-*.xml"
+          name: test_summary_super_scaffolding_${{ strategy.job-index }}_${{ inputs.use-core-repo }}.log
+          path: tmp/starter/test/reports/**/TEST-*.xml
+
+      - name: Upload Test Coverage Data
+        uses: actions/upload-artifact@v4
         if: always()
+        with:
+          name: test_coverage_super_scaffolding_${{ strategy.job-index }}_${{ inputs.use-core-repo }}.log
+          path: tmp/starter/coverage/.resultset.json
+          include-hidden-files: true