diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 72ece9dcb4..41c6770061 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,9 +2,19 @@ # # Results are uploaded to GitHub Code Scanning. # -# Due to a known issue with the CodeQL extractor when building the edk2 -# codebase on Linux systems, only Windows agents are used for build with -# the VS toolchain. +# Note: Important: This file currently only works with "CI" builds. "Platform" builds can +# be supported without much effort but that will be done in the future. +# +# Note: This workflow only supports Windows as CodeQL CLI has confirmed issues running +# against edk2-style codebases on Linux (only tested on Ubuntu). Therefore, this +# workflow is written only for Windows but could easily be adapted to run on Linux +# in the future if needed (e.g. swap out "windows" with agent OS var value, etc.) +# +# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there +# instead of the file in this repo. +# +# - Mu DevOps Repo: https://github.com/microsoft/mu_devops +# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml # # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: BSD-2-Clause-Patent @@ -14,18 +24,56 @@ name: "CodeQL" on: push: branches: - - master + - main + - release/* pull_request: branches: - - master + - main + - release/* paths-ignore: - '!**.c' - '!**.h' jobs: + gather_packages: + name: Gather Repo Packages + runs-on: ubuntu-latest + outputs: + packages: ${{ steps.generate_matrix.outputs.packages }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '>=3.11' + + - name: Generate Package Matrix + id: generate_matrix + shell: python + run: | + import os + import json + + packages = [d for d in os.listdir() if d.strip().lower().endswith('pkg')] + + # Ensure the package can actually be built + for package in packages: + if not any(file.endswith('.dsc') for file in os.listdir(package)): + packages.remove(package) + + packages.sort() + + with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: + print(f'packages={json.dumps(packages)}', file=fh) + analyze: name: Analyze - runs-on: windows-2019 + runs-on: windows-2022 + needs: + - gather_packages permissions: actions: read contents: read @@ -34,45 +82,10 @@ jobs: strategy: fail-fast: false matrix: + package: ${{ fromJson(needs.gather_packages.outputs.packages) }} include: - - Package: "ArmPkg" - ArchList: "IA32,X64" - - Package: "CryptoPkg" - ArchList: "IA32" - - Package: "CryptoPkg" - ArchList: "X64" - - Package: "DynamicTablesPkg" - ArchList: "IA32,X64" - - Package: "FatPkg" - ArchList: "IA32,X64" - - Package: "FmpDevicePkg" - ArchList: "IA32,X64" - - Package: "IntelFsp2Pkg" - ArchList: "IA32,X64" - - Package: "IntelFsp2WrapperPkg" - ArchList: "IA32,X64" - - Package: "MdeModulePkg" - ArchList: "IA32" - - Package: "MdeModulePkg" - ArchList: "X64" - - Package: "MdePkg" - ArchList: "IA32,X64" - - Package: "PcAtChipsetPkg" - ArchList: "IA32,X64" - - Package: "PrmPkg" - ArchList: "IA32,X64" - - Package: "SecurityPkg" - ArchList: "IA32,X64" - - Package: "ShellPkg" - ArchList: "IA32,X64" - - Package: "SourceLevelDebugPkg" - ArchList: "IA32,X64" - - Package: "StandaloneMmPkg" - ArchList: "IA32,X64" - - Package: "UefiCpuPkg" - ArchList: "IA32,X64" - - Package: "UnitTestFrameworkPkg" - ArchList: "IA32,X64" + - archs: IA32,X64 + - tool_chain_tag: VS2022 steps: - name: Checkout repository @@ -81,7 +94,7 @@ jobs: - name: Install Python uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '>=3.11' cache: 'pip' cache-dependency-path: 'pip-requirements.txt' @@ -105,7 +118,7 @@ jobs: from edk2toolext.invocables.edk2_ci_setup import CiSetupSettingsManager from edk2toolext.invocables.edk2_setup import SetupSettingsManager - # Find the repo CI Settings file + # Find the CI Settings file (usually in .pytool/CISettings.py) ci_settings_file = list(Path(os.environ['GITHUB_WORKSPACE']).rglob('.pytool/CISettings.py')) # Note: At this point, submodules have not been pulled, only one CI Settings file should exist @@ -136,15 +149,87 @@ jobs: print(f'ci_setup_supported={str(ci_setup_supported).lower()}', file=fh) print(f'setup_supported={str(setup_supported).lower()}', file=fh) + - name: Get Cargo Tool Details + id: get_cargo_tool_details + shell: python + run: | + import os + import requests + + GITHUB_REPO = "sagiegurari/cargo-make" + API_URL = f"https://api.github.com/repos/{GITHUB_REPO}/releases/latest" + + # Default value in case getting latest fails, cache will fall + # back on this version. + latest_cargo_make_version = "0.36.13" + response = requests.get(API_URL) + + if response.status_code == 200: + latest_cargo_make_version = response.json()["tag_name"] + else: + print("::error title=GitHub Release Error!::Failed to get latest cargo-make version!") + + cache_key = f'cargo-make-{latest_cargo_make_version}' + + with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: + print(f'cargo_bin_path={os.path.join(os.environ["USERPROFILE"], ".cargo", "bin")}', file=fh) + print(f'cargo_make_cache_key={cache_key}', file=fh) + print(f'cargo_make_version={latest_cargo_make_version}', file=fh) + + - name: Attempt to Load cargo-make From Cache + id: cargo_make_cache + uses: actions/cache@v3 + with: + path: ${{ steps.get_cargo_tool_details.outputs.cargo_bin_path }} + key: ${{ steps.get_cargo_tool_details.outputs.cargo_make_cache_key }} + + - name: Download cargo-make + if: steps.cargo_make_cache.outputs.cache-hit != 'true' + uses: robinraju/release-downloader@v1.8 + with: + repository: 'sagiegurari/cargo-make' + tag: '${{ steps.get_cargo_tool_details.outputs.cargo_make_version }}' + fileName: 'cargo-make-v${{ steps.get_cargo_tool_details.outputs.cargo_make_version }}-x86_64-pc-windows-msvc.zip' + out-file-path: 'cargo-make-download' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract cargo-make + if: steps.cargo_make_cache.outputs.cache-hit != 'true' + env: + CARGO_MAKE_VERSION: ${{ steps.get_cargo_tool_details.outputs.cargo_make_version }} + DEST_DIR: ${{steps.get_cargo_tool_details.outputs.cargo_bin_path }} + shell: python + run: | + import os + import shutil + import zipfile + from pathlib import Path + + DOWNLOAD_DIR = Path(os.environ["GITHUB_WORKSPACE"], "cargo-make-download") + ZIP_FILE_NAME = f"cargo-make-v{os.environ['CARGO_MAKE_VERSION']}-x86_64-pc-windows-msvc.zip" + ZIP_FILE_PATH = Path(DOWNLOAD_DIR, ZIP_FILE_NAME) + EXTRACT_DIR = Path(DOWNLOAD_DIR, "cargo-make-contents") + + with zipfile.ZipFile(ZIP_FILE_PATH, 'r') as zip_ref: + zip_ref.extractall(EXTRACT_DIR) + + for extracted_file in EXTRACT_DIR.iterdir(): + if extracted_file.name == "cargo-make.exe": + shutil.copy2(extracted_file, os.environ["DEST_DIR"]) + break + + - name: Rust Prep + run: rustup component add rust-src + - name: Setup if: steps.get_ci_file_operations.outputs.setup_supported == 'true' - run: stuart_setup -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 + run: stuart_setup -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.archs }} TOOL_CHAIN_TAG=${{ matrix.tool_chain_tag }} - name: Upload Setup Log As An Artifact uses: actions/upload-artifact@v3 if: (success() || failure()) && steps.get_ci_file_operations.outputs.setup_supported == 'true' with: - name: ${{ matrix.Package }}-Logs + name: ${{ matrix.package }}-Logs path: | **/SETUPLOG.txt retention-days: 7 @@ -152,34 +237,31 @@ jobs: - name: CI Setup if: steps.get_ci_file_operations.outputs.ci_setup_supported == 'true' - run: stuart_ci_setup -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 + run: stuart_ci_setup -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.archs }} TOOL_CHAIN_TAG=${{ matrix.tool_chain_tag }} - name: Upload CI Setup Log As An Artifact uses: actions/upload-artifact@v3 if: (success() || failure()) && steps.get_ci_file_operations.outputs.ci_setup_supported == 'true' with: - name: ${{ matrix.Package }}-Logs + name: ${{ matrix.package }}-Logs path: | **/CISETUP.txt retention-days: 7 if-no-files-found: ignore - name: Update - run: stuart_update -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 + run: stuart_update -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.archs }} TOOL_CHAIN_TAG=${{ matrix.tool_chain_tag }} - name: Upload Update Log As An Artifact uses: actions/upload-artifact@v3 if: success() || failure() with: - name: ${{ matrix.Package }}-Logs + name: ${{ matrix.package }}-Logs path: | **/UPDATE_LOG.txt retention-days: 7 if-no-files-found: ignore - - name: Build Tools From Source - run: python BaseTools/Edk2ToolsBuild.py -t VS2019 - - name: Find CodeQL Plugin Directory id: find_dir shell: python @@ -189,7 +271,7 @@ jobs: from pathlib import Path # Find the plugin directory that contains the CodeQL plugin - plugin_dir = list(Path(os.environ['GITHUB_WORKSPACE']).rglob('BaseTools/Plugin/CodeQL')) + plugin_dir = list(Path(os.environ['GITHUB_WORKSPACE']).rglob('.pytool/Plugin/CodeQL')) # This should only be found once if len(plugin_dir) == 1: @@ -198,7 +280,7 @@ jobs: with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: print(f'codeql_plugin_dir={plugin_dir}', file=fh) else: - print("::error title=Workspace Error!::Failed to find CodeQL plugin directory!") + print("::error title=Workspace Error!::Failed to find Mu Basecore plugin directory!") sys.exit(1) - name: Get CodeQL CLI Cache Data @@ -235,7 +317,7 @@ jobs: - name: Download CodeQL CLI if: steps.codeqlcli_cache.outputs.cache-hit != 'true' - run: stuart_update -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 --codeql + run: stuart_update -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.archs }} TOOL_CHAIN_TAG=${{ matrix.tool_chain_tag }} --codeql - name: Remove CI Plugins Irrelevant to CodeQL shell: python @@ -247,9 +329,9 @@ jobs: from pathlib import Path # Only these two plugins are needed for CodeQL - plugins_to_keep = ['CompilerPlugin'] + plugins_to_keep = ['CodeQL', 'CompilerPlugin'] - plugin_dir = Path('.pytool/Plugin').absolute() + plugin_dir = Path(os.environ['CODEQL_PLUGIN_DIR']).parent.absolute() if plugin_dir.is_dir(): for dir in plugin_dir.iterdir(): if str(dir.stem) not in plugins_to_keep: @@ -258,36 +340,13 @@ jobs: - name: CI Build env: STUART_CODEQL_PATH: ${{ steps.cache_key_gen.outputs.codeql_cli_ext_dep_dir }} - run: stuart_ci_build -c .pytool/CISettings.py -t DEBUG -p ${{ matrix.Package }} -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 --codeql - - - name: Build Cleanup - id: build_cleanup - shell: python - run: | - import os - import shutil - from pathlib import Path - - dirs_to_delete = ['ia32', 'x64', 'arm', 'aarch64'] - - def delete_dirs(path: Path): - if path.exists() and path.is_dir(): - if path.name.lower() in dirs_to_delete: - print(f'Removed {str(path)}') - shutil.rmtree(path) - return - - for child_dir in path.iterdir(): - delete_dirs(child_dir) - - build_path = Path(os.environ['GITHUB_WORKSPACE'], 'Build') - delete_dirs(build_path) + run: stuart_ci_build -c .pytool/CISettings.py -t DEBUG -p ${{ matrix.package }} -a ${{ matrix.archs }} TOOL_CHAIN_TAG=${{ matrix.tool_chain_tag }} --codeql - name: Upload Build Logs As An Artifact uses: actions/upload-artifact@v3 if: success() || failure() with: - name: ${{ matrix.Package }}-Logs + name: ${{ matrix.package }}-Logs path: | **/BUILD_REPORT.TXT **/OVERRIDELOG.TXT @@ -301,7 +360,7 @@ jobs: - name: Prepare Env Data for CodeQL Upload id: env_data env: - PACKAGE_NAME: ${{ matrix.Package }} + PACKAGE_NAME: ${{ matrix.package }} shell: python run: | import os @@ -312,27 +371,21 @@ jobs: sarif_path = os.path.join('Build', directory_name, file_name) with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: - if os.path.isfile(sarif_path): - print(f'upload_sarif_file=true', file=fh) - print(f'sarif_file_path={sarif_path}', file=fh) - else: - print(f'upload_sarif_file=false', file=fh) + print(f'sarif_file_path={sarif_path}', file=fh) - name: Upload CodeQL Results (SARIF) As An Artifact uses: actions/upload-artifact@v3 - if: steps.env_data.outputs.upload_sarif_file == 'true' with: - name: ${{ matrix.Package }}-CodeQL-SARIF + name: ${{ matrix.package }}-CodeQL-SARIF path: ${{ steps.env_data.outputs.sarif_file_path }} retention-days: 14 if-no-files-found: warn - name: Upload CodeQL Results (SARIF) To GitHub Code Scanning uses: github/codeql-action/upload-sarif@v2 - if: steps.env_data.outputs.upload_sarif_file == 'true' with: # Path to SARIF file relative to the root of the repository. sarif_file: ${{ steps.env_data.outputs.sarif_file_path }} # Optional category for the results. Used to differentiate multiple results for one commit. # Each package is a separate category. - category: ${{ matrix.Package }} + category: ${{ matrix.package }} diff --git a/.github/workflows/release-basetools.yml b/.github/workflows/release-basetools.yml index 3f9850befa..4b1a232223 100644 --- a/.github/workflows/release-basetools.yml +++ b/.github/workflows/release-basetools.yml @@ -53,7 +53,7 @@ jobs: steps: - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 @@ -103,7 +103,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download Artifacts uses: actions/download-artifact@v3