From c60ae87718686998d04bd7a4b499e7f426c6c704 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 14 Feb 2025 10:31:50 -0800 Subject: [PATCH 1/3] Try AWS CodeBuild runners for builds --- .github/workflows/kernel-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kernel-build.yml b/.github/workflows/kernel-build.yml index a91b121..92ec7fb 100644 --- a/.github/workflows/kernel-build.yml +++ b/.github/workflows/kernel-build.yml @@ -42,8 +42,8 @@ on: jobs: build: name: build for ${{ inputs.arch }} with ${{ inputs.toolchain_full }}${{ inputs.release && '-O2' || '' }} - runs-on: ${{ fromJSON(inputs.runs_on) }} - timeout-minutes: 100 + runs-on: + - codebuild-gha-runner-poc-${{ github.run_id }}-${{ github.run_attempt }} env: ARTIFACTS_ARCHIVE: "vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst" BPF_NEXT_BASE_BRANCH: 'master' From 3309d68e9eea8ee807eee645f33e42de4007eaca Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Tue, 18 Feb 2025 12:18:44 -0800 Subject: [PATCH 2/3] kernel-build: setup qemu and binfmt on the runner --- .github/workflows/kernel-build.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/kernel-build.yml b/.github/workflows/kernel-build.yml index 92ec7fb..208244f 100644 --- a/.github/workflows/kernel-build.yml +++ b/.github/workflows/kernel-build.yml @@ -98,6 +98,16 @@ jobs: llvm-version: ${{ inputs.llvm-version }} pahole: master + # We have to setup qemu+binfmt in order to enable cross-compation of selftests. + # During selftests build, freshly built bpftool is executed. + # On bare-metal and s390x hosts binfmt is pre-configured. + - name: Set up Docker + uses: docker/setup-docker-action@v4 + - name: Setup binfmt and qemu + uses: docker/setup-qemu-action@v3 + with: + image: tonistiigi/binfmt:qemu-v9.2.0 + - name: Build kernel image uses: libbpf/ci/build-linux@v3 with: From c98552537329b442202db39d3005d3a683bc7e43 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Tue, 18 Feb 2025 16:20:57 -0800 Subject: [PATCH 3/3] set-matrix: query github for runners status --- .github/scripts/matrix.py | 106 ++++++++++++++++++++++++----- .github/workflows/kernel-build.yml | 21 ++++-- .github/workflows/test.yml | 8 +++ 3 files changed, 111 insertions(+), 24 deletions(-) diff --git a/.github/scripts/matrix.py b/.github/scripts/matrix.py index 785c233..15a176d 100644 --- a/.github/scripts/matrix.py +++ b/.github/scripts/matrix.py @@ -3,9 +3,10 @@ import os import dataclasses import json +import requests from enum import Enum -from typing import Any, Dict, List, Final, Set, Union +from typing import Any, Dict, List, Final, Set, Union, Optional MANAGED_OWNER: Final[str] = "kernel-patches" MANAGED_REPOS: Final[Set[str]] = { @@ -66,6 +67,9 @@ class BuildConfig: run_veristat: bool = False parallel_tests: bool = False build_release: bool = False + build_runs_on: Optional[List[str]] = dataclasses.field( + default_factory=lambda: [DEFAULT_RUNNER] + ) @property def runs_on(self) -> List[str]: @@ -73,15 +77,6 @@ def runs_on(self) -> List[str]: return DEFAULT_SELF_HOSTED_RUNNER_TAGS + [self.arch.value] return [DEFAULT_RUNNER] - @property - def build_runs_on(self) -> List[str]: - if is_managed_repo(): - # Build s390x on x86_64 - return DEFAULT_SELF_HOSTED_RUNNER_TAGS + [ - self.arch.value == "s390x" and Arch.X86_64.value or self.arch.value, - ] - return [DEFAULT_RUNNER] - @property def tests(self) -> Dict[str, Any]: tests_list = [ @@ -158,38 +153,115 @@ def generate_test_config(test: str) -> Dict[str, Union[str, int]]: return config +def query_runners() -> Optional[List[Dict[str, Any]]]: + token = os.environ["GITHUB_TOKEN"] + headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + } + owner = os.environ["GITHUB_REPOSITORY_OWNER"] + url = f"https://api.github.com/orgs/{owner}/actions/runners" + # GitHub returns 30 runners per page, fetch all + all_runners = [] + try: + while url: + response = requests.get(url, headers=headers) + if response.status_code != 200: + return None + data = response.json() + all_runners.extend(data.get("runners", [])) + # Check for next page URL in Link header + url = None + if "Link" in response.headers: + links = requests.utils.parse_header_links(response.headers["Link"]) + for link in links: + if link["rel"] == "next": + url = link["url"] + break + return all_runners + except Exception as e: + print(f"Warning: Failed to query runner status due to exception: {e}") + return None + + +def is_x86_64_runner(runner: Dict[str, Any]) -> bool: + labels = [label["name"] for label in runner["labels"]] + for label in DEFAULT_SELF_HOSTED_RUNNER_TAGS: + if label not in labels: + return False + return "x86_64" in labels + + +def x86_64_runners_too_busy() -> bool: + + print(f"is_managed_repo(): {is_managed_repo()}") + + if not is_managed_repo(): + return False + + runners = query_runners() + if not runners: + print("Warning: failed to query runner status from GitHub") + return False + + x86_64_runners = [r for r in runners if is_x86_64_runner(r)] + n_busy = 0 + n_idle = 0 + n_offline = 0 + for runner in x86_64_runners: + if runner["status"] == "online": + if runner["busy"]: + n_busy += 1 + else: + n_idle += 1 + else: + n_offline += 1 + too_busy = n_idle == 0 or (n_busy / (n_idle + n_busy)) > 0.8 + + print(f"GitHub says we have {len(x86_64_runners)} x86_64 runners") + print(f"Busy: {n_busy}, Idle: {n_idle}, Offline: {n_offline}") + print(f"x86_64 runners too busy: {too_busy}") + + return too_busy + + if __name__ == "__main__": + + if x86_64_runners_too_busy(): + # this value is checked for in kernel-build.yml + x86_64_builder = ["codebuild"] + else: + x86_64_builder = DEFAULT_SELF_HOSTED_RUNNER_TAGS + [Arch.X86_64.value] + matrix = [ BuildConfig( arch=Arch.X86_64, toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION), run_veristat=True, parallel_tests=True, + build_runs_on=x86_64_builder, ), BuildConfig( arch=Arch.X86_64, toolchain=Toolchain(compiler=Compiler.LLVM, version=DEFAULT_LLVM_VERSION), build_release=True, + build_runs_on=x86_64_builder, ), BuildConfig( arch=Arch.X86_64, toolchain=Toolchain(compiler=Compiler.LLVM, version=18), build_release=True, + build_runs_on=x86_64_builder, ), BuildConfig( arch=Arch.AARCH64, toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION), + build_runs_on=DEFAULT_SELF_HOSTED_RUNNER_TAGS + [Arch.AARCH64.value], ), - # BuildConfig( - # arch=Arch.AARCH64, - # toolchain=Toolchain( - # compiler=Compiler.LLVM, - # version=DEFAULT_LLVM_VERSION - # ), - # ), BuildConfig( arch=Arch.S390X, toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION), + build_runs_on=x86_64_builder, ), ] diff --git a/.github/workflows/kernel-build.yml b/.github/workflows/kernel-build.yml index 208244f..93e36fe 100644 --- a/.github/workflows/kernel-build.yml +++ b/.github/workflows/kernel-build.yml @@ -42,8 +42,12 @@ on: jobs: build: name: build for ${{ inputs.arch }} with ${{ inputs.toolchain_full }}${{ inputs.release && '-O2' || '' }} - runs-on: - - codebuild-gha-runner-poc-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: >- + ${{ + contains(fromJSON(inputs.runs_on), 'codebuild') + && format('codebuild-gha-runner-poc-{0}-{1}', github.run_id, github.run_attempt) + || fromJSON(inputs.runs_on) + }} env: ARTIFACTS_ARCHIVE: "vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst" BPF_NEXT_BASE_BRANCH: 'master' @@ -54,6 +58,7 @@ jobs: KERNEL_ROOT: ${{ github.workspace }} REPO_PATH: "" REPO_ROOT: ${{ github.workspace }} + RUNNER_TYPE: ${{ contains(fromJSON(inputs.runs_on), 'codebuild') && 'codebuild' || 'self-hosted' }} steps: - uses: actions/checkout@v4 with: @@ -98,12 +103,14 @@ jobs: llvm-version: ${{ inputs.llvm-version }} pahole: master - # We have to setup qemu+binfmt in order to enable cross-compation of selftests. - # During selftests build, freshly built bpftool is executed. - # On bare-metal and s390x hosts binfmt is pre-configured. - - name: Set up Docker + # We have to setup qemu+binfmt in order to enable cross-compation of selftests. + # During selftests build, freshly built bpftool is executed. + # On self-hosted bare-metal hosts binfmt is pre-configured. + - if: ${{ env.RUNNER_TYPE == 'codebuild' }} + name: Set up docker uses: docker/setup-docker-action@v4 - - name: Setup binfmt and qemu + - if: ${{ env.RUNNER_TYPE == 'codebuild' }} + name: Setup binfmt and qemu uses: docker/setup-qemu-action@v3 with: image: tonistiigi/binfmt:qemu-v9.2.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fdf9d9f..9bf1516 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,7 @@ jobs: # This could be somehow fixed long term by making this action/workflow re-usable and letting the called # specify what to run on. runs-on: ${{ github.repository_owner == 'kernel-patches' && 'x86_64' || 'ubuntu-latest' }} + permissions: read-all outputs: build-matrix: ${{ steps.set-matrix-impl.outputs.build_matrix }} steps: @@ -27,7 +28,14 @@ jobs: sparse-checkout: | .github ci + - name: Install script dependencies + shell: bash + run: | + sudo apt-get -y update + sudo apt-get -y install python3-requests - id: set-matrix-impl + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_READ_RUNNERS }} run: | python3 .github/scripts/matrix.py