[Backport 2.1] CI/CD: Added self hosted macOS ARM runners (#4706)

liorsve · web-flow · commit 31990dea1165 · 2025-09-10T15:28:22.000+03:00
* CI: Added self hosted macOS runners (#4683) added self hosted runners for macos Signed-off-by: Lior Sventitzky <liorsve@amazon.com> * CD: removed self hosted mac arm runners from required CD platforms #4700 (#4702) removed self hosted mac from cd, reverted npm-cd Signed-off-by: Lior Sventitzky <liorsve@amazon.com> --------- Signed-off-by: Lior Sventitzky <liorsve@amazon.com>
diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json
@@ -19,6 +19,14 @@
         "PACKAGE_MANAGERS": ["pypi", "npm", "maven", "pkg_go_dev"],
         "languages": ["python", "node", "java", "go"]
     },
+    {
+        "OS": "macos",
+        "NAMED_OS": "darwin",
+        "RUNNER": ["self-hosted", "macOS", "ARM64", "ephemeral"],
+        "ARCH": "arm64",
+        "TARGET": "aarch64-apple-darwin",
+        "languages": ["python", "node", "java", "go"]
+    },
     {
         "OS": "macos",
         "NAMED_OS": "darwin",
diff --git a/.github/workflows/create-test-matrices/action.yml b/.github/workflows/create-test-matrices/action.yml
@@ -14,8 +14,11 @@ inputs:
         required: true
         type: boolean
     run-with-macos:
-        description: "Run with macos included"
-        type: boolean
+        type: choice
+        options:
+            - false
+            - use-self-hosted
+            - use-github
         default: false
     containers:
         description: "Run in containers"
@@ -83,10 +86,13 @@ runs:
               fi
                         
               # Add macOS runners if specified
-              if [[ "$RUN_WITH_MACOS" == "true" ]]; then
-                echo "Including macOS runners separately"
-                MAC_RUNNERS=$(jq --arg lang "$LANGUAGE_NAME" -c '[.[] | select(.languages? and any(.languages[] == $lang; .) and '"$CONDITION"' and .TARGET == "aarch64-apple-darwin")]' < .github/json_matrices/build-matrix.json)
-                            
+              if [[ "$RUN_WITH_MACOS" == "use-self-hosted" ]]; then
+                echo "Including only self-hosted macOS runners"
+                MAC_RUNNERS=$(jq --arg lang "$LANGUAGE_NAME" -c '[.[] | select(.languages? and any(.languages[] == $lang; .) and '"$CONDITION"' and .TARGET == "aarch64-apple-darwin" and (.RUNNER == ["self-hosted","macOS","ARM64","ephemeral"]))]' < .github/json_matrices/build-matrix.json)
+                FINAL_MATRIX=$(echo "$BASE_MATRIX" "$MAC_RUNNERS" | jq -sc 'add')
+              elif [[ "$RUN_WITH_MACOS" == "use-github" ]]; then
+                echo "Including only GitHub-hosted macOS runners"
+                MAC_RUNNERS=$(jq --arg lang "$LANGUAGE_NAME" -c '[.[] | select(.languages? and any(.languages[] == $lang; .) and '"$CONDITION"' and .TARGET == "aarch64-apple-darwin" and .RUNNER == "macos-15")]' < .github/json_matrices/build-matrix.json)
                 FINAL_MATRIX=$(echo "$BASE_MATRIX" "$MAC_RUNNERS" | jq -sc 'add')
               else
                 FINAL_MATRIX="$BASE_MATRIX"
diff --git a/.github/workflows/full-matrix-tests.yml b/.github/workflows/full-matrix-tests.yml
@@ -14,8 +14,12 @@ on:
                 default: true
 
             run-with-macos:
-                description: "Run with macos included (only when necessary)"
-                type: boolean
+                description: "Run with macos inclauded (only when necessary)"
+                type: choice
+                options:
+                    - false
+                    - use-self-hosted
+                    - use-github
                 default: false
 
             run-modules-tests:
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -43,7 +43,11 @@ on:
                 default: false
             run-with-macos:
                 description: "Run with macos included (only when needed)"
-                type: boolean
+                type: choice
+                options:
+                    - false
+                    - use-self-hosted
+                    - use-github
                 default: false
             rc-version:
                 required: false
@@ -61,8 +65,8 @@ on:
     workflow_call:
         inputs:
             run-with-macos:
-                description: "Run with macos included (only when needed)"
-                type: boolean
+                description: "Run with macos included (only when necessary)"
+                type: string
                 default: false
 
 concurrency:
@@ -92,7 +96,7 @@ jobs:
                   language-name: go
                   # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow
                   run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }}
-                  run-with-macos: ${{ github.event.inputs.run-with-macos == 'true' }}
+                  run-with-macos: ${{ github.event.inputs.run-with-macos }}
 
     test-go:
         name: Go Tests - ${{ matrix.go }}, EngineVersion - ${{ matrix.engine.version }}, Target - ${{ matrix.host.TARGET }}
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
@@ -44,7 +44,11 @@ on:
                 default: false
             run-with-macos:
                 description: "Run with macos included (only when necessary)"
-                type: boolean
+                type: choice
+                options:
+                    - false
+                    - use-self-hosted
+                    - use-github
                 default: false
             name:
                 required: false
@@ -59,7 +63,7 @@ on:
         inputs:
             run-with-macos:
                 description: "Run with macos included (only when necessary)"
-                type: boolean
+                type: string
                 default: false
 
 concurrency:
@@ -85,7 +89,7 @@ jobs:
                   language-name: java
                   # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow
                   run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }}
-                  run-with-macos: ${{ (github.event.inputs.run-with-macos == 'true') }}
+                  run-with-macos: ${{ (github.event.inputs.run-with-macos) }}
 
     test-java:
         name: Java Tests - ${{ matrix.java }}, EngineVersion - ${{ matrix.engine.version }}, Target - ${{ matrix.host.TARGET }}
diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml
@@ -42,7 +42,11 @@ on:
                 default: false
             run-with-macos:
                 description: "Run with macos included (only when necessary)"
-                type: boolean
+                type: choice
+                options:
+                    - false
+                    - use-self-hosted
+                    - use-github
                 default: false
             name:
                 required: false
@@ -57,7 +61,7 @@ on:
         inputs:
             run-with-macos:
                 description: "Run with macos included (only when necessary)"
-                type: boolean
+                type: string
                 default: false
 
 concurrency:
@@ -86,7 +90,7 @@ jobs:
               with:
                   language-name: node
                   run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }}
-                  run-with-macos: ${{ github.event.run-with-macos == 'true' }}
+                  run-with-macos: ${{ github.event.run-with-macos }}
 
     test-node:
         name: Node Tests - ${{ matrix.node }}, EngineVersion - ${{ matrix.engine.version }}, Target - ${{ matrix.host.TARGET }}
diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml
@@ -149,7 +149,7 @@ jobs:
               id: load-platform-matrix
               shell: bash
               run: |
-                  # Filter entries with npm in PACKAGE_MANAGERS and use CD_RUNNER if available
+                  # Filter entries with npm in PACKAGE_MANAGERS and use CD_RUNNER if available, replace "ephemeral" with "persistent" in RUNNER
                   export PLATFORM_MATRIX=$(jq 'map(
                       select(.PACKAGE_MANAGERS != null and (.PACKAGE_MANAGERS | contains(["npm"])))
                       | .runner = (
@@ -271,7 +271,7 @@ jobs:
             - name: Upload Native Modules
               uses: actions/upload-artifact@v4
               with:
-                  name: bindings-${{ matrix.TARGET }}
+                  name: bindings-${{ matrix.TARGET }}-${{ matrix.RUNNER }}
                   path: ./node/${{ matrix.TARGET }}
                   retention-days: 1
                   if-no-files-found: error
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -45,7 +45,11 @@ on:
                 default: false
             run-with-macos:
                 description: "Run with macos included (only when necessary)"
-                type: boolean
+                type: choice
+                options:
+                    - false
+                    - use-self-hosted
+                    - use-github
                 default: false
             name:
                 required: false
@@ -70,7 +74,7 @@ on:
         inputs:
             run-with-macos:
                 description: "Run with macos included (only when necessary)"
-                type: boolean
+                type: string
                 default: false
 
 concurrency:
@@ -89,7 +93,7 @@ run-name:
 env:
     # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow
     RUN_FULL_MATRIX: ${{ (github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule') }}
-    RUN_WITH_MACOS: ${{ (github.event.inputs.run-with-macos == 'true') }}
+    RUN_WITH_MACOS: ${{ (github.event.inputs.run-with-macos) }}
     RUN_SYNC_ONLY: ${{ (github.event.inputs.run_sync_tests == 'true' && github.event.inputs.run_async_tests == 'false') }}
     RUN_ASYNC_ONLY: ${{ (github.event.inputs.run_async_tests == 'true' && github.event.inputs.run_sync_tests == 'false') }}
 
@@ -107,7 +111,7 @@ jobs:
               with:
                   language-name: python
                   run-full-matrix: ${{ env.RUN_FULL_MATRIX == 'true' }}
-                  run-with-macos: ${{ env.RUN_WITH_MACOS == 'true' }}
+                  run-with-macos: ${{ env.RUN_WITH_MACOS }}
 
     test-python:
         name: Python Tests - ${{ matrix.python }}, EngineVersion - ${{ matrix.engine.version }}, Target - ${{ matrix.host.TARGET }}
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -44,7 +44,11 @@ on:
                 default: false
             run-with-macos:
                 description: "Run with macos included (only when necessary)"
-                type: boolean
+                type: choice
+                options:
+                    - false
+                    - use-self-hosted
+                    - use-github
                 default: false
             name:
                 required: false
@@ -55,7 +59,7 @@ on:
         inputs:
             run-with-macos:
                 description: "Run with macos included (only when necessary)"
-                type: boolean
+                type: string
                 default: false
 
 concurrency:
@@ -87,7 +91,7 @@ jobs:
                   language-name: rust
                   # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow
                   run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }}
-                  run-with-macos: ${{ github.event.inputs.run-with-macos == 'true' }}
+                  run-with-macos: ${{ github.event.inputs.run-with-macos }}
 
     tests:
         runs-on: ${{ matrix.host.RUNNER }}
