chore(ci): Better local running (#17432)

* Fixed import-merge syntax

* Add conditional publishing steps

* Set runner image by env

* Only check existence of relevant branches

* Don't run locally

* Names and debug

* Improve get-cache

* Revert "Improve get-cache"

This reverts commit fd15e54.

* Add cache-url input and only set when not local

* pipefail by default

* Install java if missing (for codegen

* Only home paths, and quoting

* // For local development

* Home debug

* Only override cypress cache when no env variable is set

* Fall back to all affected if stickman fails

* Allow and handle failure in permission check

* Don't collect failures

* Revert codemagic.yaml to main

* Comment, and improve readability

* Revert image config

* Revert default shell

* Subshell node list

* debug checking out branches

* Fetch branch and comment

* Create main branches as well

* Remove duplicate branch creation

* Remove java install step

* Refactor 'Preparing HEAD and BASE tags'

* Only try to create branches if they don't exist

* Revert "Only try to create branches if they don't exist"

This reverts commit c0b1146.

* Spacing

* Verify or create branches

* Remove // For local development

* Comment on Cypress override

* Reduce diff for 00_prepare-base-tags.sh

* pullrequest.yml cleanup

* Always wait for collector (revert to main)

* Support localrun & less diff

* Re-add octokit auth

* Debug and pipefail get-cache exporting

* Revert actions/ to main

* Remove pipefail for export

* Rabbit review

* Move get-cache required env vars back to github-script

* Revert checking out relevant branches to main, but with force

* checkout, but preserve local changes

* localrun cleanup

* Only use github.event.localrun for local condition

* Use if for permission, not continue-on-error (reviewfix)

* Don't run permission check locally

.github/actions/get-cache/action.yml

@@ -4,6 +4,9 @@ inputs:
   github-token:
     description: 'GitHub token'
     required: true
+  cache-url:
+    description: 'Cache URL'
+    default: 'https://cache.dev01.devland.is/'
   enable-cache:
     description: 'Enable cache'
     default: ''
@@ -18,12 +21,15 @@ runs:
   using: 'composite'
   steps:
     - name: Adding required env vars
+      # Skip when running locally
+      if: ${{ !github.event.localrun }}
       uses: actions/github-script@v7
       env:
+        cache-url: ${{ inputs.cache-url }}
         github-token: ${{ inputs.GITHUB_TOKEN }}
       with:
         script: |
-          core.exportVariable('ACTIONS_CACHE_URL', 'https://cache.dev01.devland.is/')
+          core.exportVariable('ACTIONS_CACHE_URL', process.env['cache-url'])
           core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN'])
           core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL'])
     - name: Get cache
@@ -35,8 +41,8 @@ runs:
         NODE_OPTIONS: --max-old-space-size=8192
       run: |
         # This is because of a weird bug where Github would write as root to .cache
-        export COREPACK_HOME=/github/home/.corepack-cache
-        export CYPRESS_CACHE_FOLDER=/github/home/.cypress-cache
+        export COREPACK_HOME="$HOME/.corepack-cache"
+        export CYPRESS_CACHE_FOLDER="$HOME/.cypress-cache"
         echo "COREPACK_HOME=$COREPACK_HOME" >> $GITHUB_ENV
         echo "CYPRESS_CACHE_FOLDER=$CYPRESS_CACHE_FOLDER" >> $GITHUB_ENV
         cd scripts/ci/cache

.github/workflows/pullrequest.yml

@@ -40,9 +40,17 @@ jobs:
       BUILD_CHUNKS: ${{ steps.build_projects.outputs.CHUNKS }}
       CACHE_KEYS: ${{ steps.get-cache.outputs.keys }}
     steps:
+      # Creates homedir if missing (e.g. when set to /tmp/runner/ when running locally with act)
+      - name: Debug home etc.
+        run: |
+          echo "User: HOME=$HOME, PWD=$PWD, id=$(id)"
+          mkdir -p "$HOME"
+          ls -lah "$HOME"
+
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
       - uses: actions/setup-node@v4
         with:
           node-version-file: 'package.json'
@@ -54,13 +62,17 @@ jobs:
         run: |
           set -euo pipefail
           node -v
-          ls -l `which node`
+          yarn --version
+          ls -l "$(which node)"
+
       - name: Checking out relevant branches
         run: |
           set -euo pipefail
-          git checkout "$GITHUB_HEAD_REF"
-          git checkout "$GITHUB_BASE_REF"
-          git checkout "$GITHUB_SHA"
+          set -x
+
+          git checkout -m "$GITHUB_HEAD_REF"
+          git checkout -m "$GITHUB_BASE_REF"
+          git checkout -m "$GITHUB_SHA"
           git config --global user.email "ci@island.is"
           git config --global user.name "CI Bot"
 
@@ -74,6 +86,8 @@ jobs:
       # https://github.blog/changelog/2020-10-08-github-actions-ability-to-change-retention-days-for-artifacts-and-logs/
       - name: Keep PR run event
         uses: actions/upload-artifact@b18b1d32f3f31abcdc29dee3f2484801fe7822f4
+        # Don't run this step locally
+        if: ${{ !github.event.localrun }}
         with:
           name: pr-event
           path: event.json
@@ -88,7 +102,12 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           enable-cache: 'node_modules,cypress,generated-files'
 
-      - run: |
+      - name: Preparing HEAD and BASE tags
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          HTML_URL: ${{ github.event.pull_request.html_url }}
+          ISSUE_REPORTING_SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILD_ISSUES_REPORTING_WEBHOOK_URL }}
+        run: |
           set -euo pipefail
           echo "HEAD=$GITHUB_SHA" >> "$GITHUB_ENV"
           export HEAD_REF="$GITHUB_HEAD_REF"
@@ -99,26 +118,23 @@ jobs:
           source ./scripts/ci/00_prepare-base-tags.sh "$(git merge-base HEAD "$GITHUB_BASE_REF")"
           git checkout "$GITHUB_SHA"
           echo "BASE=$BASE" >> "$GITHUB_ENV"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          HTML_URL: ${{ github.event.pull_request.html_url }}
-          ISSUE_REPORTING_SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILD_ISSUES_REPORTING_WEBHOOK_URL }}
-        name: Preparing HEAD and BASE tags
 
       - name: License audit Node modules
         run: ./scripts/ci/20_license-audit.sh
 
       - name: Check user permissions
         uses: actions-cool/check-user-permission@v2
         id: check-permission
+        # This fails locally without a token, and only prevents "admin" label usage
+        if: ${{ !github.event.localrun }}
 
       - name: Set magic env if test-everything label is set
-        if: ${{ contains(github.event.pull_request.labels.*.name, 'test everything') && steps.check-permission.outputs['user-permission'] == 'admin' }}
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'test everything') && steps.check-permission.outcome == 'success' && steps.check-permission.outputs['user-permission'] == 'admin' }}
         run: |
           echo "AFFECTED_ALL=7913-$GITHUB_HEAD_REF" >> "$GITHUB_ENV"
 
       - name: Warn if user does not have the required permissions
-        if: ${{ contains(github.event.pull_request.labels.*.name, 'test everything') && steps.check-permission.outputs['user-permission'] != 'admin' }}
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'test everything') && steps.check-permission.outcome == 'success'&& steps.check-permission.outputs['user-permission'] != 'admin' }}
         run: |
           echo "## WARN permissions" >> "$GITHUB_STEP_SUMMARY"
           echo "User '$GITHUB_ACTOR' does not have the required permissions to apply the 'test everything' label" >> "$GITHUB_STEP_SUMMARY"
@@ -205,7 +221,8 @@ jobs:
           keys: ${{ needs.prepare.outputs.CACHE_KEYS }}
           enable-cache: 'node_modules,cypress,generated-files'
 
-      - uses: ./.github/actions/unit-test
+      - name: Run unit tests
+        uses: ./.github/actions/unit-test
         with:
           dd-api-key: '${{ secrets.DD_API_KEY }}'
           codecov-token: ${{ secrets.CODECOV_TOKEN }}

scripts/ci/00_prepare-base-tags.sh

@@ -4,32 +4,48 @@ set -euxo pipefail
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 ROOT="$DIR/../.."
 
+# Copy the built action to a temp directory
 tempRepo=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
 cp -r "$ROOT/.github/actions/dist/." "$tempRepo"
 
-LAST_GOOD_BUILD=$(DEBUG="*,-simple-git" REPO_ROOT="$ROOT" node "$tempRepo/main.js")
+# Get last good build via GitHub API, forcing full rebuild if it fails
+if ! LAST_GOOD_BUILD=$(DEBUG="*,-simple-git" REPO_ROOT="$ROOT" node "$tempRepo/main.js"); then
+  >&2 echo "Stickman failed 😢, continuing with full rebuild"
+  LAST_GOOD_BUILD="full_rebuild_needed"
+fi
+>&2 echo "Stickman done"
+
+# Check if the script failed or if a full rebuild is needed
 if echo "$LAST_GOOD_BUILD" | grep -q 'full_rebuild_needed'; then
+  >&2 echo "No last good build found"
   export NX_AFFECTED_ALL=true
   echo "NX_AFFECTED_ALL=$NX_AFFECTED_ALL" >>"$GITHUB_ENV"
   exit 0
 fi
-echo "Stickman done"
+
+# Extract details from the last good build
 LAST_GOOD_BUILD_SHA=$(echo "$LAST_GOOD_BUILD" | jq -r '.sha')
 LAST_GOOD_BUILD_BRANCH=$(echo "$LAST_GOOD_BUILD" | jq -r '.branch')
 LAST_GOOD_BUILD_RUN_NUMBER=$(echo "$LAST_GOOD_BUILD" | jq -r '.run_number')
 BUILD_REF=$(echo "$LAST_GOOD_BUILD" | jq -r '.ref')
 >&2 echo "Last successful build is with SHA '$LAST_GOOD_BUILD_SHA', branch '$LAST_GOOD_BUILD_BRANCH' and number '$LAST_GOOD_BUILD_RUN_NUMBER'"
+
+# Log commits between the last good build and the current build
 if [[ "$BUILD_REF" != "$LAST_GOOD_BUILD_SHA" ]]; then
   echo "This will be an incremental build from a previous successful run in this PR. See parents of the commit below."
   git log -1 "$BUILD_REF"
 fi
+
+# Generate a Docker tag based on the last good build details
 LAST_GOOD_BUILD_DOCKER_BRANCH_TAG=$(echo "${LAST_GOOD_BUILD_BRANCH}" | tr "/." "-")
 export LAST_GOOD_BUILD_DOCKER_TAG=${LAST_GOOD_BUILD_DOCKER_BRANCH_TAG:0:45}_${LAST_GOOD_BUILD_SHA:0:10}_${LAST_GOOD_BUILD_RUN_NUMBER}
+
+# If the build reference is invalid, report an error to Slack
 if [[ "$BUILD_REF" == "null" || "$BUILD_REF" == "" ]]; then
   curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"Change detection failed for $HTML_URL\"}" "$ISSUE_REPORTING_SLACK_WEBHOOK_URL"
   exit 1
-else
-  BASE="$BUILD_REF"
 fi
-export BASE
+export BASE="$BUILD_REF"
+
+# Log the Docker tag of the last successful build
 >&2 echo "Last successful docker tag '$LAST_GOOD_BUILD_DOCKER_TAG'"

scripts/ci/cache/__config.mjs

@@ -33,7 +33,9 @@ export const ENABLED_MODULES = (process.env[ENV_ENABLED_CACHE] || '')
     return a
   }, {})
 
-export const cypressPath = '/github/home/.cypress-cache'
+// Override default Cypress cache; the default (~/.cache) could be unwritable
+export const cypressPath =
+  process.env.CYPRESS_CACHE_FOLDER || `${process.env.HOME}/.cypress-cache`
 export const cacheSuccess = JSON.parse(process.env[ENV_CACHE_SUCCESS] ?? '{}')
 export const initCache = process.env[ENV_INIT_CACHE] === 'true'