Add a fallback to free GH runners, when there is a problem with the self-hosted ones

.github/workflows/build.yml

@@ -46,6 +46,7 @@ jobs:
  # Please declare any key you added below in build_steps.yaml's dummy matrix as well to aid the linting tools
  - linux_matrix:
  - os: linux
+ distro: ubuntu-latest
  cmake_preset_prefix: linux
  cibw_build_suffix: manylinux_x86_64
  build_dir: /tmp/cpp_build
@@ -63,6 +64,7 @@ jobs:
  - /:/mnt
  windows_matrix:
  - os: windows
+ distro: windows-latest
  cmake_preset_prefix: windows-cl
  cibw_build_suffix: win_amd64
  build_dir: C:/cpp_build
@@ -153,7 +155,7 @@ jobs:
  cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
  cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
  matrix: ${{needs.common_config.outputs.linux_matrix}}
- distro: compile-on-ec2
+ compile-override: compile-on-ec2
 
  build-python-wheels-linux:
  # Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels
@@ -190,7 +192,7 @@ jobs:
  matrix: ${{toJson(matrix.matrix_override)}}
  python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
  persistent_storage: ${{inputs.persistent_storage}}
- distro: compile-on-ec2
+ compile-override: compile-on-ec2
 
  cpp-test-windows:
  needs: [common_config]
@@ -201,7 +203,6 @@ jobs:
  job_type: cpp-tests
  cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
  matrix: ${{needs.common_config.outputs.windows_matrix}}
- distro: windows-latest
 
  build-python-wheels-windows:
  needs: [common_config]
@@ -220,7 +221,6 @@ jobs:
  cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
  matrix: ${{needs.common_config.outputs.windows_matrix}}
  persistent_storage: ${{ inputs.persistent_storage }}
- distro: windows-latest
 
  persistent_storage_verify_linux:
  needs: [common_config, build-python-wheels-linux, build-python-wheels-windows]

.github/workflows/build_steps.yml

@@ -5,7 +5,7 @@ on:
  job_type: {required: true, type: string, description: Selects the steps to enable}
  cmake_preset_type: {required: true, type: string, description: release/debug}
  matrix: {required: true, type: string, description: JSON string to feed into the matrix}
- distro:  {required: true, type: string, description: Label of the runner that will be used for running the workflow}
+ compile-override: {required: false, type: string, description: Parameter to override the agent that is used for compiling e.g. compile-on-ec2}
  cibw_image_tag: {required: false, type: string, description: Linux only. As built by cibw_docker_image.yml workflow}
  cibw_version: {required: false, type: string, description: build-python-wheels only. Must match the cibw_image_tag}
  python_deps_ids: {default: '[""]', type: string, description: build-python-wheels test matrix parameter. JSON string.}
@@ -18,7 +18,7 @@ on:
  clear: {default: 1, type: number, description: Controls wheather we will clear the libraries in the versions stores that we test against}
 jobs:
  start_ec2_runner:
- if: inputs.distro == 'compile-on-ec2'
+ if: inputs.compile-override == 'compile-on-ec2'
  uses: ./.github/workflows/ec2_runner_jobs.yml
  secrets: inherit
  with:
@@ -28,7 +28,6 @@ jobs:
  needs: [start_ec2_runner]
  if: |
  always() &&
- !failure() &&
  !cancelled()
  strategy:
  matrix:
@@ -49,7 +48,7 @@ jobs:
  include:
  - ${{fromJSON(inputs.matrix)[0]}} # The items after 0 are for tests only
 
- runs-on: ${{ needs.start_ec2_runner.outputs.label || inputs.distro}}
+ runs-on: ${{ needs.start_ec2_runner.result != 'failure' && needs.start_ec2_runner.outputs.label || matrix.distro}}
  container: ${{ (matrix.os == 'linux' && inputs.job_type != 'build-python-wheels') && matrix.container || null}}
  env:
  SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} # Setting this env var enables the caching
@@ -220,7 +219,7 @@ jobs:
  needs: [start_ec2_runner, compile]
  if: |
  always() &&
- inputs.distro == 'compile-on-ec2'
+ inputs.compile-override == 'compile-on-ec2'
  uses: ./.github/workflows/ec2_runner_jobs.yml
  secrets: inherit
  with:
@@ -243,7 +242,7 @@ jobs:
  include:
  ${{fromJSON(inputs.matrix)}}
  name: ${{matrix.type}}${{matrix.python_deps_id}}
- runs-on: ${{inputs.distro != 'compile-on-ec2' && inputs.distro || 'ubuntu-latest'}}
+ runs-on: ${{matrix.distro}}
  container: ${{matrix.os == 'linux' && needs.compile.outputs.manylinux_image || null}}
  defaults:
  run: {shell: bash}

.github/workflows/build_with_conda.yml

@@ -19,8 +19,11 @@ jobs:
  job_type: start
 
  linux:
+ if: |
+ always() &&
+ !cancelled()
  needs: [start_ec2_runner]
- runs-on: ${{ needs.start_ec2_runner.outputs.label }}
+ runs-on: ${{ needs.start_ec2_runner.result != 'failure' && needs.start_ec2_runner.outputs.label || 'ubuntu-latest'}}
  services:
  mongodb:
  image: mongo:4.4

.github/workflows/ec2_runner_jobs.yml

@@ -19,6 +19,7 @@ jobs:
  if: inputs.job_type == 'start'
  name: Start self-hosted EC2 runner
  runs-on: ubuntu-latest
+ continue-on-error: true
  outputs:
  label: ${{ steps.start-ec2-runner.outputs.label }}
  ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
@@ -45,6 +46,7 @@ jobs:
  name: Stop self-hosted EC2 runner
  if: inputs.job_type == 'stop'
  runs-on: ubuntu-latest
+ continue-on-error: true
  steps:
  - name: Configure AWS credentials
  uses: aws-actions/configure-aws-credentials@v1