From 679a472500eb7278152b6d731e2439b21ceab981 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 18 Apr 2023 15:54:48 -0700 Subject: [PATCH 01/41] Update coredistools from LLVM 13.0.1 to 16.0.1 1. LLVM doesn't release a full set of binary drops, so we need to build llvm-tblgen on some platforms, namely Mac. 2. Build using standard CBL-Mariner Docker containers (used by dotnet/runtime as well). This also converts linux-x64 builds to be container-based cross builds. --- build-coredistools.sh | 40 ++++++++---- build-tblgen.sh | 116 +++++++++++++++++++++++++++++++++++ coredistools.yml | 75 +++++++++------------- doc/building-coredistools.md | 14 ++--- eng/download-llvm-release.py | 4 ++ 5 files changed, 184 insertions(+), 65 deletions(-) create mode 100644 build-tblgen.sh diff --git a/build-coredistools.sh b/build-coredistools.sh index da44cad0..92758501 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -13,12 +13,29 @@ EnsureCrossRootfsDirectoryExists () { CMakeOSXArchitectures= LLVMTargetsToBuild="AArch64;ARM;X86" +# Figure out which `strip` to use. Prefer `llvm-strip` if it is available. +# `llvm-strip` is available in CBL-Mariner container; `strip` is available on macOS. +StripTool=$(command -v llvm-strip) +if [ -z "$StripTool" ]; then + StripTool=$(command -v strip) + if [ -z "$StripTool" ]; then + echo "Strip tool not found" + exit 1 + fi +fi + +TblGenTool=$(command -v llvm-tblgen) +if [ -z "$TblGenTool" ]; then + echo "llvm-tblgen tool not found" + exit 1 +fi + case "$TargetOSArchitecture" in linux-arm) CMakeCrossCompiling=ON LLVMDefaultTargetTriple=thumbv7-linux-gnueabihf LLVMHostTriple=arm-linux-gnueabihf - LLVMTargetsToBuild=ARM + LLVMTargetsToBuild="ARM" EnsureCrossRootfsDirectoryExists ;; @@ -29,8 +46,9 @@ case "$TargetOSArchitecture" in ;; linux-x64) - CMakeCrossCompiling=OFF + CMakeCrossCompiling=ON LLVMHostTriple=x86_64-linux-gnu + EnsureCrossRootfsDirectoryExists ;; linux-loongarch64) @@ -63,7 +81,7 @@ SourcesDirectory=$RootDirectory/src BinariesDirectory=$RootDirectory/obj/$TargetOSArchitecture StagingDirectory=$RootDirectory/artifacts/$TargetOSArchitecture -which cmake >/dev/null 2>&1 +command -v cmake >/dev/null 2>&1 if [ "$?" -ne 0 ]; then echo "ERROR: cmake is not found in the PATH" @@ -81,20 +99,20 @@ if [ -z "$CrossRootfsDirectory" ]; then -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ - -DCMAKE_C_COMPILER=$(which clang) \ + -DCMAKE_C_COMPILER=$(command -v clang) \ -DCMAKE_C_FLAGS="-target $LLVMHostTriple" \ - -DCMAKE_CXX_COMPILER=$(which clang++) \ + -DCMAKE_CXX_COMPILER=$(command -v clang++) \ -DCMAKE_CXX_FLAGS="-target $LLVMHostTriple" \ -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ -DCMAKE_OSX_ARCHITECTURES=$CMakeOSXArchitectures \ - -DCMAKE_STRIP=$(which strip) \ + -DCMAKE_STRIP=$StripTool \ -DLLVM_DEFAULT_TARGET_TRIPLE=$LLVMDefaultTargetTriple \ -DLLVM_ENABLE_TERMINFO=OFF \ -DLLVM_EXTERNAL_PROJECTS=coredistools \ -DLLVM_EXTERNAL_COREDISTOOLS_SOURCE_DIR=$SourcesDirectory/coredistools \ -DLLVM_HOST_TRIPLE=$LLVMHostTriple \ -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_TABLEGEN=$(which llvm-tblgen) \ + -DLLVM_TABLEGEN=$TblGenTool \ -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ $SourcesDirectory/llvm-project/llvm @@ -103,21 +121,21 @@ else -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ - -DCMAKE_C_COMPILER=$(which clang) \ + -DCMAKE_C_COMPILER=$(command -v clang) \ -DCMAKE_C_FLAGS="-target $LLVMHostTriple --sysroot=$CrossRootfsDirectory" \ - -DCMAKE_CXX_COMPILER=$(which clang++) \ + -DCMAKE_CXX_COMPILER=$(command -v clang++) \ -DCMAKE_CXX_FLAGS="-target $LLVMHostTriple --sysroot=$CrossRootfsDirectory" \ -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ - -DCMAKE_STRIP=/usr/$LLVMHostTriple/bin/strip \ + -DCMAKE_STRIP=$StripTool \ -DLLVM_DEFAULT_TARGET_TRIPLE=$LLVMDefaultTargetTriple \ -DLLVM_ENABLE_TERMINFO=OFF \ -DLLVM_EXTERNAL_PROJECTS=coredistools \ -DLLVM_EXTERNAL_COREDISTOOLS_SOURCE_DIR=$SourcesDirectory/coredistools \ -DLLVM_HOST_TRIPLE=$LLVMHostTriple \ -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_TABLEGEN=$(which llvm-tblgen) \ + -DLLVM_TABLEGEN=$TblGenTool \ -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ $SourcesDirectory/llvm-project/llvm diff --git a/build-tblgen.sh b/build-tblgen.sh new file mode 100644 index 00000000..c137062b --- /dev/null +++ b/build-tblgen.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# Build the Linux/Mac llvm-tblgen tool. Note that this will be run during the +# Linux/Mac coredistools build. So, we only build the versions that we'll use +# during those builds. Thus, we need a linux-x64 version (used for +# linux-x64, linux-arm, linux-arm64 builds) and osx-x64 version (used for +# osx-x64 and osx-arm64 builds). +# +# The linux-x64 build is itself a cross-build, using CBL-Mariner container to build. + +TargetOSArchitecture=$1 +CrossRootfsDirectory=$2 + +EnsureCrossRootfsDirectoryExists () { + if [ ! -d "$CrossRootfsDirectory" ]; then + echo "Invalid or unspecified CrossRootfsDirectory: $CrossRootfsDirectory" + exit 1 + fi +} + +CMakeOSXArchitectures= +LLVMTargetsToBuild="AArch64;ARM;X86" + +case "$TargetOSArchitecture" in + linux-x64) + CMakeCrossCompiling=ON + LLVMHostTriple=x86_64-linux-gnu + EnsureCrossRootfsDirectoryExists + ;; + + linux-loongarch64) + CMakeCrossCompiling=OFF + LLVMHostTriple=loongarch64-linux-gnu + LLVMTargetsToBuild="LoongArch" + ;; + + osx-x64) + CMakeCrossCompiling=OFF + CMakeOSXArchitectures=x86_64 + LLVMHostTriple=x86_64-apple-darwin + ;; + + *) + echo "Unknown target OS and architecture: $TargetOSArchitecture" + exit 1 +esac + +RootDirectory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +SourcesDirectory=$RootDirectory/src +BinariesDirectory=$RootDirectory/obj +StagingDirectory=$RootDirectory/bin + +command -v cmake >/dev/null 2>&1 + +if [ "$?" -ne 0 ]; then + echo "ERROR: cmake is not found in the PATH" + exit 1 +fi + +if [ ! -d $BinariesDirectory ]; then + mkdir -p $BinariesDirectory +fi + +pushd "$BinariesDirectory" + +if [ -z "$CrossRootfsDirectory" ]; then + cmake \ + -G "Unix Makefiles" \ + -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ + -DCMAKE_C_COMPILER=$(command -v clang) \ + -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_INSTALL_PREFIX=$RootDirectory \ + -DCMAKE_OSX_ARCHITECTURES=$CMakeOSXArchitectures \ + -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ + $SourcesDirectory/llvm-project/llvm +else + cmake \ + -G "Unix Makefiles" \ + -DCMAKE_INSTALL_PREFIX=$RootDirectory \ + -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ + -DCMAKE_C_COMPILER=$(command -v clang) \ + -DCMAKE_C_FLAGS="--sysroot=$CrossRootfsDirectory" \ + -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_CXX_FLAGS="--sysroot=$CrossRootfsDirectory" \ + -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ + -DCMAKE_INSTALL_PREFIX=$RootDirectory \ + -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ + -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ + $SourcesDirectory/llvm-project/llvm +fi + +popd + +if [ "$?" -ne 0 ]; then + echo "ERROR: cmake exited with code $1" + exit 1 +fi + +cmake \ + --build $BinariesDirectory \ + --target llvm-tblgen \ + --config Release + +if [ "$?" -ne 0 ]; then + echo "ERROR: cmake exited with code $1" + exit 1 +fi + +if [ ! -d $StagingDirectory ]; then + mkdir -p $StagingDirectory +fi + +# Copy llvm-tblgen from BinariesDirectory to StagingDirectory +find $BinariesDirectory -name llvm-tblgen + +exit 0 diff --git a/coredistools.yml b/coredistools.yml index 776a6646..f1df37ff 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -28,17 +28,22 @@ trigger: resources: containers: - - container: ubuntu-18.04-arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666 - - container: ubuntu-18.04-arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 + - container: linux_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-20230414190614-8100bf7 + - container: linux_arm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-20230414190614-8100bf7 + - container: linux_arm64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-20230414190614-8100bf7 variables: LLVMRepositoryUri: https://github.com/llvm/llvm-project.git LLVMSourceBundle: llvm-project.bundle - LLVMSourceVersion: llvmorg-13.0.1 + LLVMSourceVersion: llvmorg-16.0.1 jobs: + +################################ Check out LLVM source tree; publish to artifacts for future jobs + - job: checkout_llvm displayName: Checkout LLVM @@ -61,6 +66,8 @@ jobs: artifact: $(LLVMSourceBundle) displayName: Publish LLVM bundle +################################ Cross-build coredistools for linux-x64, linux-arm, linux-arm64 in CBL-Mariner Docker container + - job: crossbuild_coredistools_linux dependsOn: checkout_llvm displayName: Build coredistools Linux @@ -72,13 +79,18 @@ jobs: strategy: matrix: + x64: + ContainerImage: linux_x64 + CrossRootfsDirectory: /crossrootfs/x64 + TargetOSArchitecture: linux-x64 + arm: - ContainerImage: ubuntu-18.04-arm + ContainerImage: linux_arm CrossRootfsDirectory: /crossrootfs/arm TargetOSArchitecture: linux-arm arm64: - ContainerImage: ubuntu-18.04-arm64 + ContainerImage: linux_arm64 CrossRootfsDirectory: /crossrootfs/arm64 TargetOSArchitecture: linux-arm64 @@ -88,11 +100,6 @@ jobs: steps: - template: /eng/download-checkout-llvm.yml - - template: /eng/download-llvm-release.yml - parameters: - os: linux - release: $(LLVMSourceVersion) - - script: ./build-coredistools.sh $(TargetOSArchitecture) $(CrossRootfsDirectory) displayName: Build coredistools @@ -100,33 +107,7 @@ jobs: artifact: coredistools-$(TargetOSArchitecture) displayName: Publish coredistools -- job: build_coredistools_linux_x64 - dependsOn: checkout_llvm - displayName: Build coredistools Linux x64 - - pool: - vmImage: ubuntu-18.04 - - variables: - TargetOSArchitecture: linux-x64 - - workspace: - clean: all - - steps: - - template: /eng/download-checkout-llvm.yml - - - template: /eng/download-llvm-release.yml - parameters: - os: linux - release: $(LLVMSourceVersion) - - - script: ./build-coredistools.sh $(TargetOSArchitecture) - displayName: Build coredistools - - - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.so - artifact: coredistools-$(TargetOSArchitecture) - displayName: Publish coredistools +################################ Build coredistools for macos-x64, macos-arm64 - job: build_coredistools_macos dependsOn: checkout_llvm @@ -139,10 +120,10 @@ jobs: matrix: x64: TargetOSArchitecture: osx-x64 - VMImage: macOS-10.15 + VMImage: macOS-latest arm64: TargetOSArchitecture: osx-arm64 - VMImage: macOS-11 + VMImage: macOS-latest workspace: clean: all @@ -150,11 +131,6 @@ jobs: steps: - template: /eng/download-checkout-llvm.yml - - template: /eng/download-llvm-release.yml - parameters: - os: macos - release: $(LLVMSourceVersion) - - script: ./build-coredistools.sh $(TargetOSArchitecture) displayName: Build coredistools @@ -162,6 +138,8 @@ jobs: artifact: coredistools-$(TargetOSArchitecture) displayName: Publish coredistools +################################ Build llvm-tblgen on Windows + - job: build_tblgen_windows dependsOn: checkout_llvm displayName: Build llvm-tblgen Windows @@ -182,6 +160,8 @@ jobs: artifact: tblgen-windows displayName: Publish llvm-tblgen +################################ Build coredistools for win-x64, win-x86, win-arm64 + - job: build_coredistools_windows dependsOn: - checkout_llvm @@ -220,10 +200,11 @@ jobs: artifact: coredistools-$(TargetOSArchitecture) displayName: Publish coredistools +################################ Build coredistools NuGet packages + - job: build_coredistools_nuget_packages dependsOn: - crossbuild_coredistools_linux - - build_coredistools_linux_x64 - build_coredistools_macos - build_coredistools_windows displayName: Build coredistools NuGet packages diff --git a/doc/building-coredistools.md b/doc/building-coredistools.md index dd99f6bb..63ec0f32 100644 --- a/doc/building-coredistools.md +++ b/doc/building-coredistools.md @@ -10,7 +10,7 @@ cd jitutils 2. Checkout the LLVM project repository: ``` -git clone --depth 1 --branch llvmorg-13.0.1 https://github.com/llvm/llvm-project.git src\llvm-project +git clone --depth 1 --branch llvmorg-16.0.1 https://github.com/llvm/llvm-project.git src\llvm-project ``` 4. Build `llvm-tblgen.exe`: @@ -59,24 +59,24 @@ cd jitutils 2. Checkout the LLVM project repository: ``` -git clone --depth 1 --branch llvmorg-13.0.1 https://github.com/llvm/llvm-project.git src/llvm-project +git clone --depth 1 --branch llvmorg-16.0.1 https://github.com/llvm/llvm-project.git src/llvm-project ``` 3. Download LLVM release from GitHub: ``` -python3 eng/download-llvm-release.py -release llvmorg-13.0.1 -os linux +python3 eng/download-llvm-release.py -release llvmorg-16.0.1 -os linux ``` 4. Locate under the current directory file `llvm-tblgen` ``` find -name llvm-tblgen -./clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04/bin/llvm-tblgen +./clang+llvm-16.0.1-x86_64-linux-gnu-ubuntu-18.04/bin/llvm-tblgen ``` and add its parent directory location to the `PATH`: ``` -export PATH=$(pwd)/clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +export PATH=$(pwd)/clang+llvm-16.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ``` 5. Build `libcoredistools.so` for Linux x64: @@ -95,14 +95,14 @@ find ./artifacts -name libcoredistools.so ``` docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 -export PATH=$(pwd)/clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +export PATH=$(pwd)/clang+llvm-16.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ./build-coredistools.sh linux-arm64 /crossrootfs/arm64 ``` 7. Build `libcoredistools.so` for Linux arm under Docker: ``` docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666 -export PATH=$(pwd)/clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +export PATH=$(pwd)/clang+llvm-16.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ./build-coredistools.sh linux-arm /crossrootfs/arm ``` diff --git a/eng/download-llvm-release.py b/eng/download-llvm-release.py index 78fbac96..bf4fdfe2 100755 --- a/eng/download-llvm-release.py +++ b/eng/download-llvm-release.py @@ -15,6 +15,10 @@ 'llvmorg-13.0.1': { 'linux': 'https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04.tar.xz', 'macos': 'https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/clang+llvm-13.0.1-x86_64-apple-darwin.tar.xz' + }, + 'llvmorg-16.0.1': { + 'linux': 'https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.1/clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz', + 'macos': 'https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.1/clang+llvm-13.0.1-x86_64-apple-darwin.tar.xz' } } From 0cff1f382e5636f8ba2d03a6203bc6f25d98658b Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 18 Apr 2023 20:20:52 -0700 Subject: [PATCH 02/41] Update documentation Also, fix one compilation bug to build coredistools.cpp with LLVM 16.0.1. --- build-coredistools.cmd | 1 + doc/building-coredistools.md | 21 +++++++++++++-------- src/coredistools/coredistools.cpp | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/build-coredistools.cmd b/build-coredistools.cmd index 613ba6d4..ffe11e5a 100644 --- a/build-coredistools.cmd +++ b/build-coredistools.cmd @@ -16,6 +16,7 @@ if /i "%TargetOSArchitecture%" == "win-arm64" ( set LLVMTargetsToBuild=ARM;X86 ) else ( echo ERROR: Unknown target OS and architecture: %TargetOSArchitecture% + echo Use one of win-arm64, win-x64, win-x86. exit /b 1 ) diff --git a/doc/building-coredistools.md b/doc/building-coredistools.md index 63ec0f32..844fcd3f 100644 --- a/doc/building-coredistools.md +++ b/doc/building-coredistools.md @@ -8,22 +8,26 @@ git clone https://github.com/dotnet/jitutils.git cd jitutils ``` -2. Checkout the LLVM project repository: +2. Checkout the LLVM project repository into a subdirectory named src/llvm-project: ``` git clone --depth 1 --branch llvmorg-16.0.1 https://github.com/llvm/llvm-project.git src\llvm-project ``` -4. Build `llvm-tblgen.exe`: +3. Build `llvm-tblgen.exe`: ``` build-tblgen.cmd ``` -5. Add the `bin` subdirectory to the `PATH`: +This builds llvm-tblgen.exe and puts it in the `bin` subdirectory. + +4. Add the `bin` subdirectory to the `PATH`: ``` set "PATH=%cd%\bin;%PATH%" -```` +``` -6. Build `coredistools.dll` for a combination of target OS and architecture. +This puts the just built lldb-tblgen.exe on the `PATH`. + +5. Build `coredistools.dll` for a combination of target OS and architecture. For example, the following command will result in `coredistools.dll` binary that can be run on Windows x64: ``` @@ -37,7 +41,7 @@ F:\echesako\git\jitutils\artifacts\win-x64\bin\coredistools.dll F:\echesako\git\jitutils\artifacts\win-x64\lib\coredistools.lib ``` -7. Build Windows x86, Windows ARM and Windows ARM64 binaries: +6. Build Windows x86, Windows ARM and Windows ARM64 binaries: ``` build-coredistools.cmd win-x86 build-coredistools.cmd win-arm @@ -49,7 +53,7 @@ build-coredistools.cmd win-arm64 The `build-coredistools.cmd` script is set up to build a Release build. To create a Debug build with a PDB file, for debugging, change the `--config Release` line to `--config Debug`. -## Building on Linux +## Building on Linux / Mac 1. Checkout the jitutils repository: ``` @@ -93,6 +97,8 @@ find ./artifacts -name libcoredistools.so 6. Build `libcoredistools.so` for Linux arm64 under Docker: +TODO: update these Docker images with CBL-Mariner images. + ``` docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 export PATH=$(pwd)/clang+llvm-16.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH @@ -105,4 +111,3 @@ docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/c export PATH=$(pwd)/clang+llvm-16.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ./build-coredistools.sh linux-arm /crossrootfs/arm ``` - diff --git a/src/coredistools/coredistools.cpp b/src/coredistools/coredistools.cpp index f302b205..9648c28f 100644 --- a/src/coredistools/coredistools.cpp +++ b/src/coredistools/coredistools.cpp @@ -27,7 +27,7 @@ #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetRegistry.h" +#include "llvm/MC/TargetRegistry.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/DataTypes.h" From 8e59b24686ac42351214c31600ce2453bf1a3368 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Wed, 19 Apr 2023 22:48:55 -0700 Subject: [PATCH 03/41] Fix macOS build Build llvm-tblgen from source on Mac --- build-tblgen.sh | 4 +++- coredistools.yml | 33 ++++++++++++++++++++++++- eng/download-llvm-release.py | 45 ----------------------------------- eng/download-llvm-release.yml | 12 ---------- 4 files changed, 35 insertions(+), 59 deletions(-) mode change 100644 => 100755 build-tblgen.sh delete mode 100755 eng/download-llvm-release.py delete mode 100644 eng/download-llvm-release.yml diff --git a/build-tblgen.sh b/build-tblgen.sh old mode 100644 new mode 100755 index c137062b..1655c601 --- a/build-tblgen.sh +++ b/build-tblgen.sh @@ -66,6 +66,7 @@ pushd "$BinariesDirectory" if [ -z "$CrossRootfsDirectory" ]; then cmake \ -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ -DCMAKE_C_COMPILER=$(command -v clang) \ -DCMAKE_CXX_COMPILER=$(command -v clang++) \ @@ -76,6 +77,7 @@ if [ -z "$CrossRootfsDirectory" ]; then else cmake \ -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$RootDirectory \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ -DCMAKE_C_COMPILER=$(command -v clang) \ @@ -111,6 +113,6 @@ if [ ! -d $StagingDirectory ]; then fi # Copy llvm-tblgen from BinariesDirectory to StagingDirectory -find $BinariesDirectory -name llvm-tblgen +find $BinariesDirectory -name llvm-tblgen -type f -exec cp -v {} $StagingDirectory \; exit 0 diff --git a/coredistools.yml b/coredistools.yml index f1df37ff..a03e9c14 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -107,10 +107,34 @@ jobs: artifact: coredistools-$(TargetOSArchitecture) displayName: Publish coredistools +################################ Build llvm-tblgen on Mac (x64) + +- job: build_tblgen_macos + dependsOn: checkout_llvm + displayName: Build llvm-tblgen macOS + + pool: + vmImage: macOS-latest + + workspace: + clean: all + + steps: + - template: /eng/download-checkout-llvm.yml + + - script: ./build-tblgen.sh osx-x64 + displayName: Build llvm-tblgen + + - publish: $(Build.SourcesDirectory)/bin/llvm-tblgen + artifact: tblgen-macos + displayName: Publish llvm-tblgen + ################################ Build coredistools for macos-x64, macos-arm64 - job: build_coredistools_macos - dependsOn: checkout_llvm + dependsOn: + - checkout_llvm + - build_tblgen_macos displayName: Build coredistools macOS pool: @@ -131,6 +155,13 @@ jobs: steps: - template: /eng/download-checkout-llvm.yml + - download: current + artifact: tblgen-macos + displayName: Download llvm-tblgen + + - script: 'echo ##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-macos' + displayName: Add llvm-tblgen to the PATH + - script: ./build-coredistools.sh $(TargetOSArchitecture) displayName: Build coredistools diff --git a/eng/download-llvm-release.py b/eng/download-llvm-release.py deleted file mode 100755 index bf4fdfe2..00000000 --- a/eng/download-llvm-release.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the .NET Foundation under one or more agreements. -# The .NET Foundation licenses this file to you under the MIT license. -# -import argparse -import io -import os -import sys -import tarfile -from urllib import request -from urllib.error import URLError, HTTPError - -Release_urls = { - 'llvmorg-13.0.1': { - 'linux': 'https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04.tar.xz', - 'macos': 'https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/clang+llvm-13.0.1-x86_64-apple-darwin.tar.xz' - }, - 'llvmorg-16.0.1': { - 'linux': 'https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.1/clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz', - 'macos': 'https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.1/clang+llvm-13.0.1-x86_64-apple-darwin.tar.xz' - } -} - -def download_llvm_release(release_url, output_dir): - try: - with request.urlopen(release_url) as response: - downloaded_bytes = response.read() - except (URLError, HTTPError) as err: - print(err) - sys.exit(1) - - with io.BytesIO(downloaded_bytes) as downloaded_fileobj: - with tarfile.open(fileobj=downloaded_fileobj, mode='r:xz') as archive: - archive.extractall(path=output_dir) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-release', required=True, choices=Release_urls.keys()) - parser.add_argument('-os', required=True, choices=['linux', 'macos']) - parser.add_argument('-output-dir', dest='output_dir', default=os.getcwd()) - args = parser.parse_args() - - release_url = Release_urls[args.release][args.os] - download_llvm_release(release_url, args.output_dir) diff --git a/eng/download-llvm-release.yml b/eng/download-llvm-release.yml deleted file mode 100644 index 6ad1cd62..00000000 --- a/eng/download-llvm-release.yml +++ /dev/null @@ -1,12 +0,0 @@ -parameters: -- name: os - type: string -- name: release - type: string - -steps: -- script: python3 $(Build.SourcesDirectory)/eng/download-llvm-release.py -os ${{ parameters.os }} -release ${{ parameters.release }} -output-dir $(Pipeline.Workspace) - displayName: Download LLVM release from GitHub - -- script: echo '##vso[task.prependpath]'$(find $(Pipeline.Workspace) -name "clang+llvm-*" -type d)/bin - displayName: Add Clang and LLVM to the PATH From 6d78ba633376d72734f2301caca21df0cbaac0cb Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Wed, 19 Apr 2023 23:00:00 -0700 Subject: [PATCH 04/41] Minor build fixes for CBL-Mariner It still doesn't build correctly. --- build-coredistools.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/build-coredistools.sh b/build-coredistools.sh index 92758501..f3635e26 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -95,14 +95,15 @@ fi pushd "$BinariesDirectory" if [ -z "$CrossRootfsDirectory" ]; then + BUILD_FLAGS="-target $LLVMHostTriple" cmake \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ -DCMAKE_C_COMPILER=$(command -v clang) \ - -DCMAKE_C_FLAGS="-target $LLVMHostTriple" \ -DCMAKE_CXX_COMPILER=$(command -v clang++) \ - -DCMAKE_CXX_FLAGS="-target $LLVMHostTriple" \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ -DCMAKE_OSX_ARCHITECTURES=$CMakeOSXArchitectures \ -DCMAKE_STRIP=$StripTool \ @@ -117,14 +118,20 @@ if [ -z "$CrossRootfsDirectory" ]; then -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ $SourcesDirectory/llvm-project/llvm else + BUILD_FLAGS="--sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" + # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" + # CBL-Mariner doesn't seem to have libgcc_s.so in a standard place, so as a hack, add + # -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5 + # where it does exist. cmake \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ -DCMAKE_C_COMPILER=$(command -v clang) \ - -DCMAKE_C_FLAGS="-target $LLVMHostTriple --sysroot=$CrossRootfsDirectory" \ -DCMAKE_CXX_COMPILER=$(command -v clang++) \ - -DCMAKE_CXX_FLAGS="-target $LLVMHostTriple --sysroot=$CrossRootfsDirectory" \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5" \ -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ From 563c7dbf9b2b9fa81a4d1c67f354c55a0ffff71c Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 20 Apr 2023 10:17:31 -0700 Subject: [PATCH 05/41] TEST: display paths; debug Mac path issue --- coredistools.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/coredistools.yml b/coredistools.yml index a03e9c14..369667bb 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -155,13 +155,40 @@ jobs: steps: - template: /eng/download-checkout-llvm.yml + - script: | + echo "PATH=$PATH" + echo "Pipeline.Workspace=$(Pipeline.Workspace)" + echo "Files in $(Pipeline.Workspace):" + ls $(Pipeline.Workspace) + echo "Files in $(Pipeline.Workspace)/tblgen-macos" + ls $(Pipeline.Workspace)/tblgen-macos + displayName: Display data before download llvm-tblgen + - download: current artifact: tblgen-macos displayName: Download llvm-tblgen + - script: | + echo "PATH=$PATH" + echo "Pipeline.Workspace=$(Pipeline.Workspace)" + echo "Files in $(Pipeline.Workspace):" + ls $(Pipeline.Workspace) + echo "Files in $(Pipeline.Workspace)/tblgen-macos" + ls $(Pipeline.Workspace)/tblgen-macos + displayName: Display data before set PATH + - script: 'echo ##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-macos' displayName: Add llvm-tblgen to the PATH + - script: | + echo "PATH=$PATH" + echo "Pipeline.Workspace=$(Pipeline.Workspace)" + echo "Files in $(Pipeline.Workspace):" + ls $(Pipeline.Workspace) + echo "Files in $(Pipeline.Workspace)/tblgen-macos" + ls $(Pipeline.Workspace)/tblgen-macos + displayName: Display data before build coredistools + - script: ./build-coredistools.sh $(TargetOSArchitecture) displayName: Build coredistools From d315458768dd89afb7434d695893366da0e3ae82 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 20 Apr 2023 14:06:08 -0700 Subject: [PATCH 06/41] More diagnostic output --- coredistools.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index 369667bb..e982208b 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -159,9 +159,7 @@ jobs: echo "PATH=$PATH" echo "Pipeline.Workspace=$(Pipeline.Workspace)" echo "Files in $(Pipeline.Workspace):" - ls $(Pipeline.Workspace) - echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls $(Pipeline.Workspace)/tblgen-macos + ls -AFG $(Pipeline.Workspace) displayName: Display data before download llvm-tblgen - download: current @@ -172,9 +170,9 @@ jobs: echo "PATH=$PATH" echo "Pipeline.Workspace=$(Pipeline.Workspace)" echo "Files in $(Pipeline.Workspace):" - ls $(Pipeline.Workspace) + ls -AFG $(Pipeline.Workspace) echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls $(Pipeline.Workspace)/tblgen-macos + ls -AFG $(Pipeline.Workspace)/tblgen-macos || echo "Not found" displayName: Display data before set PATH - script: 'echo ##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-macos' @@ -184,12 +182,21 @@ jobs: echo "PATH=$PATH" echo "Pipeline.Workspace=$(Pipeline.Workspace)" echo "Files in $(Pipeline.Workspace):" - ls $(Pipeline.Workspace) + ls -AFG $(Pipeline.Workspace) echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls $(Pipeline.Workspace)/tblgen-macos + ls -AFG $(Pipeline.Workspace)/tblgen-macos || echo "Not found" displayName: Display data before build coredistools - - script: ./build-coredistools.sh $(TargetOSArchitecture) + #- script: ./build-coredistools.sh $(TargetOSArchitecture) + # displayName: Build coredistools + - script: | + echo "PATH=$PATH" + echo "Pipeline.Workspace=$(Pipeline.Workspace)" + echo "Files in $(Pipeline.Workspace):" + ls -AFG $(Pipeline.Workspace) + echo "Files in $(Pipeline.Workspace)/tblgen-macos" + ls -AFG $(Pipeline.Workspace)/tblgen-macos || echo "Not found" + ./build-coredistools.sh $(TargetOSArchitecture) displayName: Build coredistools - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.dylib From d0959db6c3162e6ab9c34392ebd2e555abcb953c Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 20 Apr 2023 14:27:52 -0700 Subject: [PATCH 07/41] Try again --- coredistools.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index e982208b..a887fb6b 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -190,13 +190,13 @@ jobs: #- script: ./build-coredistools.sh $(TargetOSArchitecture) # displayName: Build coredistools - script: | - echo "PATH=$PATH" - echo "Pipeline.Workspace=$(Pipeline.Workspace)" - echo "Files in $(Pipeline.Workspace):" - ls -AFG $(Pipeline.Workspace) - echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls -AFG $(Pipeline.Workspace)/tblgen-macos || echo "Not found" - ./build-coredistools.sh $(TargetOSArchitecture) + echo "PATH=$PATH" + echo "Pipeline.Workspace=$(Pipeline.Workspace)" + echo "Files in $(Pipeline.Workspace):" + ls -AFG $(Pipeline.Workspace) + echo "Files in $(Pipeline.Workspace)/tblgen-macos" + ls -AFG $(Pipeline.Workspace)/tblgen-macos || echo "Not found" + ./build-coredistools.sh $(TargetOSArchitecture) displayName: Build coredistools - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.dylib From 2efca46759f7515dab0faff84fd9e4977f7e6083 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 20 Apr 2023 15:07:10 -0700 Subject: [PATCH 08/41] Try again --- coredistools.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/coredistools.yml b/coredistools.yml index a887fb6b..38d6c31f 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -175,7 +175,10 @@ jobs: ls -AFG $(Pipeline.Workspace)/tblgen-macos || echo "Not found" displayName: Display data before set PATH - - script: 'echo ##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-macos' + - script: | + echo "PATH=$PATH" + echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-macos" + echo "PATH=$PATH" displayName: Add llvm-tblgen to the PATH - script: | From fea8d48aa67724b54328e62454f266ebfd0f03fa Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 20 Apr 2023 15:48:41 -0700 Subject: [PATCH 09/41] Make llvm-tblgen executable --- coredistools.yml | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index 38d6c31f..efeacd5c 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -159,7 +159,7 @@ jobs: echo "PATH=$PATH" echo "Pipeline.Workspace=$(Pipeline.Workspace)" echo "Files in $(Pipeline.Workspace):" - ls -AFG $(Pipeline.Workspace) + ls -lAF $(Pipeline.Workspace) displayName: Display data before download llvm-tblgen - download: current @@ -170,9 +170,22 @@ jobs: echo "PATH=$PATH" echo "Pipeline.Workspace=$(Pipeline.Workspace)" echo "Files in $(Pipeline.Workspace):" - ls -AFG $(Pipeline.Workspace) + ls -lAF $(Pipeline.Workspace) echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls -AFG $(Pipeline.Workspace)/tblgen-macos || echo "Not found" + ls -lAF $(Pipeline.Workspace)/tblgen-macos || echo "Not found" + displayName: Display data before make llvm-tblgen executable + + - script: | + chmod +x $(Pipeline.Workspace)/tblgen-macos/llvm-tblgen + displayName: Make llvm-tblgen executable + + - script: | + echo "PATH=$PATH" + echo "Pipeline.Workspace=$(Pipeline.Workspace)" + echo "Files in $(Pipeline.Workspace):" + ls -lAF $(Pipeline.Workspace) + echo "Files in $(Pipeline.Workspace)/tblgen-macos" + ls -lAF $(Pipeline.Workspace)/tblgen-macos || echo "Not found" displayName: Display data before set PATH - script: | @@ -185,9 +198,9 @@ jobs: echo "PATH=$PATH" echo "Pipeline.Workspace=$(Pipeline.Workspace)" echo "Files in $(Pipeline.Workspace):" - ls -AFG $(Pipeline.Workspace) + ls -lAF $(Pipeline.Workspace) echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls -AFG $(Pipeline.Workspace)/tblgen-macos || echo "Not found" + ls -lAF $(Pipeline.Workspace)/tblgen-macos || echo "Not found" displayName: Display data before build coredistools #- script: ./build-coredistools.sh $(TargetOSArchitecture) @@ -196,9 +209,9 @@ jobs: echo "PATH=$PATH" echo "Pipeline.Workspace=$(Pipeline.Workspace)" echo "Files in $(Pipeline.Workspace):" - ls -AFG $(Pipeline.Workspace) + ls -lAF $(Pipeline.Workspace) echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls -AFG $(Pipeline.Workspace)/tblgen-macos || echo "Not found" + ls -lAF $(Pipeline.Workspace)/tblgen-macos || echo "Not found" ./build-coredistools.sh $(TargetOSArchitecture) displayName: Build coredistools From 5a00612bbd27cf203a09f328bcd07c71b41cb5f9 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 20 Apr 2023 17:03:51 -0700 Subject: [PATCH 10/41] Build llvm-tblgen from source for Linux --- coredistools.yml | 90 +++++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index efeacd5c..d0e13f52 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -9,6 +9,7 @@ pr: - build-coredistools.cmd - build-coredistools.sh - build-tblgen.cmd + - build-tblgen.sh - coredistools.yml - pack-coredistools.cmd @@ -66,10 +67,36 @@ jobs: artifact: $(LLVMSourceBundle) displayName: Publish LLVM bundle +################################ Build llvm-tblgen on Linux (x64) + +- job: build_tblgen_linux + dependsOn: checkout_llvm + displayName: Build llvm-tblgen linux + + container: linux_x64 + + pool: + vmImage: ubuntu-latest + + workspace: + clean: all + + steps: + - template: /eng/download-checkout-llvm.yml + + - script: ./build-tblgen.sh linux-x64 /crossrootfs/x64 + displayName: Build llvm-tblgen + + - publish: $(Build.SourcesDirectory)/bin/llvm-tblgen + artifact: tblgen-linux + displayName: Publish llvm-tblgen + ################################ Cross-build coredistools for linux-x64, linux-arm, linux-arm64 in CBL-Mariner Docker container - job: crossbuild_coredistools_linux - dependsOn: checkout_llvm + dependsOn: + - checkout_llvm + - build_tblgen_linux displayName: Build coredistools Linux container: $[ variables['ContainerImage'] ] @@ -100,7 +127,22 @@ jobs: steps: - template: /eng/download-checkout-llvm.yml - - script: ./build-coredistools.sh $(TargetOSArchitecture) $(CrossRootfsDirectory) + - download: current + artifact: tblgen-linux + displayName: Download llvm-tblgen + + - script: | + chmod +x $(Pipeline.Workspace)/tblgen-linux/llvm-tblgen + displayName: Make llvm-tblgen executable + + - script: | + ls -lAF $(Pipeline.Workspace) + ls -lAF $(Pipeline.Workspace)/tblgen-linux + echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" + displayName: Add llvm-tblgen to the PATH + + - script: | + ./build-coredistools.sh $(TargetOSArchitecture) $(CrossRootfsDirectory) displayName: Build coredistools - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.so @@ -155,63 +197,19 @@ jobs: steps: - template: /eng/download-checkout-llvm.yml - - script: | - echo "PATH=$PATH" - echo "Pipeline.Workspace=$(Pipeline.Workspace)" - echo "Files in $(Pipeline.Workspace):" - ls -lAF $(Pipeline.Workspace) - displayName: Display data before download llvm-tblgen - - download: current artifact: tblgen-macos displayName: Download llvm-tblgen - - script: | - echo "PATH=$PATH" - echo "Pipeline.Workspace=$(Pipeline.Workspace)" - echo "Files in $(Pipeline.Workspace):" - ls -lAF $(Pipeline.Workspace) - echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls -lAF $(Pipeline.Workspace)/tblgen-macos || echo "Not found" - displayName: Display data before make llvm-tblgen executable - - script: | chmod +x $(Pipeline.Workspace)/tblgen-macos/llvm-tblgen displayName: Make llvm-tblgen executable - script: | - echo "PATH=$PATH" - echo "Pipeline.Workspace=$(Pipeline.Workspace)" - echo "Files in $(Pipeline.Workspace):" - ls -lAF $(Pipeline.Workspace) - echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls -lAF $(Pipeline.Workspace)/tblgen-macos || echo "Not found" - displayName: Display data before set PATH - - - script: | - echo "PATH=$PATH" echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-macos" - echo "PATH=$PATH" displayName: Add llvm-tblgen to the PATH - script: | - echo "PATH=$PATH" - echo "Pipeline.Workspace=$(Pipeline.Workspace)" - echo "Files in $(Pipeline.Workspace):" - ls -lAF $(Pipeline.Workspace) - echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls -lAF $(Pipeline.Workspace)/tblgen-macos || echo "Not found" - displayName: Display data before build coredistools - - #- script: ./build-coredistools.sh $(TargetOSArchitecture) - # displayName: Build coredistools - - script: | - echo "PATH=$PATH" - echo "Pipeline.Workspace=$(Pipeline.Workspace)" - echo "Files in $(Pipeline.Workspace):" - ls -lAF $(Pipeline.Workspace) - echo "Files in $(Pipeline.Workspace)/tblgen-macos" - ls -lAF $(Pipeline.Workspace)/tblgen-macos || echo "Not found" ./build-coredistools.sh $(TargetOSArchitecture) displayName: Build coredistools From e0924822b44a211b6af9900c4995728932c7cfd7 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 20 Apr 2023 17:14:29 -0700 Subject: [PATCH 11/41] Support building tblgen for Linux-x64 --- build-tblgen.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/build-tblgen.sh b/build-tblgen.sh index 1655c601..a0e4ef49 100755 --- a/build-tblgen.sh +++ b/build-tblgen.sh @@ -64,28 +64,36 @@ fi pushd "$BinariesDirectory" if [ -z "$CrossRootfsDirectory" ]; then + BUILD_FLAGS="" cmake \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ -DCMAKE_C_COMPILER=$(command -v clang) \ -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_INSTALL_PREFIX=$RootDirectory \ -DCMAKE_OSX_ARCHITECTURES=$CMakeOSXArchitectures \ -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ $SourcesDirectory/llvm-project/llvm else + BUILD_FLAGS="--sysroot=$CrossRootfsDirectory" + # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" + # CBL-Mariner doesn't seem to have libgcc_s.so in a standard place, so as a hack, add + # -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5 + # where it does exist. cmake \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$RootDirectory \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ -DCMAKE_C_COMPILER=$(command -v clang) \ - -DCMAKE_C_FLAGS="--sysroot=$CrossRootfsDirectory" \ -DCMAKE_CXX_COMPILER=$(command -v clang++) \ - -DCMAKE_CXX_FLAGS="--sysroot=$CrossRootfsDirectory" \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5" \ -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ - -DCMAKE_INSTALL_PREFIX=$RootDirectory \ -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ $SourcesDirectory/llvm-project/llvm From 6447421c03604b825d3bd65363149006b3de0b62 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 20 Apr 2023 18:22:36 -0700 Subject: [PATCH 12/41] Support non-CBL-Mariner builds Add options to scripts for non-CBL-Mariner builds. Revert coredistools.yml to building using non-CBL-Mariner (the CBL-Mariner logic is commented out). This means bringing back the native (non-cross) linux-x64 build. --- build-coredistools.sh | 37 ++++++++++++++++-- build-tblgen.sh | 30 +++++++++++++-- coredistools.yml | 88 ++++++++++++++++++++++++++++++++++++------- 3 files changed, 135 insertions(+), 20 deletions(-) diff --git a/build-coredistools.sh b/build-coredistools.sh index f3635e26..a94f3d58 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -3,6 +3,9 @@ TargetOSArchitecture=$1 CrossRootfsDirectory=$2 +# Set this to 1 to build using CBL-Mariner +CrossBuildUsingMariner=0 + EnsureCrossRootfsDirectoryExists () { if [ ! -d "$CrossRootfsDirectory" ]; then echo "Invalid or unspecified CrossRootfsDirectory: $CrossRootfsDirectory" @@ -46,9 +49,13 @@ case "$TargetOSArchitecture" in ;; linux-x64) - CMakeCrossCompiling=ON LLVMHostTriple=x86_64-linux-gnu - EnsureCrossRootfsDirectoryExists + if [ $CrossBuildUsingMariner -eq 1 ]; then + CMakeCrossCompiling=ON + EnsureCrossRootfsDirectoryExists + else + CMakeCrossCompiling=OFF + fi ;; linux-loongarch64) @@ -117,7 +124,7 @@ if [ -z "$CrossRootfsDirectory" ]; then -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ $SourcesDirectory/llvm-project/llvm -else +elif [ $CrossBuildUsingMariner -eq 1 ]; then BUILD_FLAGS="--sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" # CBL-Mariner doesn't seem to have libgcc_s.so in a standard place, so as a hack, add @@ -146,6 +153,30 @@ else -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ $SourcesDirectory/llvm-project/llvm +else + BUILD_FLAGS="--sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" + cmake \ + -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ + -DCMAKE_C_COMPILER=$(command -v clang) \ + -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ + -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ + -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ + -DCMAKE_STRIP=$StripTool \ + -DLLVM_DEFAULT_TARGET_TRIPLE=$LLVMDefaultTargetTriple \ + -DLLVM_ENABLE_TERMINFO=OFF \ + -DLLVM_EXTERNAL_PROJECTS=coredistools \ + -DLLVM_EXTERNAL_COREDISTOOLS_SOURCE_DIR=$SourcesDirectory/coredistools \ + -DLLVM_HOST_TRIPLE=$LLVMHostTriple \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_TABLEGEN=$TblGenTool \ + -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ + -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ + $SourcesDirectory/llvm-project/llvm fi popd diff --git a/build-tblgen.sh b/build-tblgen.sh index a0e4ef49..5c61b7cf 100755 --- a/build-tblgen.sh +++ b/build-tblgen.sh @@ -6,11 +6,14 @@ # linux-x64, linux-arm, linux-arm64 builds) and osx-x64 version (used for # osx-x64 and osx-arm64 builds). # -# The linux-x64 build is itself a cross-build, using CBL-Mariner container to build. +# The linux-x64 build is itself a cross-build when using CBL-Mariner container to build. TargetOSArchitecture=$1 CrossRootfsDirectory=$2 +# Set this to 1 to build using CBL-Mariner +CrossBuildUsingMariner=0 + EnsureCrossRootfsDirectoryExists () { if [ ! -d "$CrossRootfsDirectory" ]; then echo "Invalid or unspecified CrossRootfsDirectory: $CrossRootfsDirectory" @@ -23,9 +26,13 @@ LLVMTargetsToBuild="AArch64;ARM;X86" case "$TargetOSArchitecture" in linux-x64) - CMakeCrossCompiling=ON LLVMHostTriple=x86_64-linux-gnu - EnsureCrossRootfsDirectoryExists + if [ $CrossBuildUsingMariner -eq 1 ]; then + CMakeCrossCompiling=ON + EnsureCrossRootfsDirectoryExists + else + CMakeCrossCompiling=OFF + fi ;; linux-loongarch64) @@ -77,7 +84,7 @@ if [ -z "$CrossRootfsDirectory" ]; then -DCMAKE_OSX_ARCHITECTURES=$CMakeOSXArchitectures \ -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ $SourcesDirectory/llvm-project/llvm -else +elif [ $CrossBuildUsingMariner -eq 1 ]; then BUILD_FLAGS="--sysroot=$CrossRootfsDirectory" # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" # CBL-Mariner doesn't seem to have libgcc_s.so in a standard place, so as a hack, add @@ -97,6 +104,21 @@ else -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ $SourcesDirectory/llvm-project/llvm +else + BUILD_FLAGS="--sysroot=$CrossRootfsDirectory" + cmake \ + -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$RootDirectory \ + -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ + -DCMAKE_C_COMPILER=$(command -v clang) \ + -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ + -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ + -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ + $SourcesDirectory/llvm-project/llvm fi popd diff --git a/coredistools.yml b/coredistools.yml index d0e13f52..96f666c6 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -1,3 +1,5 @@ +# Changes to build with CBL-Mariner are commented out and marked "CBL-Mariner" + pr: branches: include: @@ -27,14 +29,23 @@ trigger: - coredistools.yml - pack-coredistools.cmd +# CBL-Mariner: +#resources: +# containers: +# - container: linux_x64 +# image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-20230414190614-8100bf7 +# - container: linux_arm +# image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-20230414190614-8100bf7 +# - container: linux_arm64 +# image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-20230414190614-8100bf7 + +# Use the following for legacy Ubuntu 18.04 based Arm/Arm64 cross build (x64 build is not cross) resources: containers: - - container: linux_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-20230414190614-8100bf7 - - container: linux_arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-20230414190614-8100bf7 - - container: linux_arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-20230414190614-8100bf7 + - container: ubuntu-18.04-arm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666 + - container: ubuntu-18.04-arm64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 variables: LLVMRepositoryUri: https://github.com/llvm/llvm-project.git @@ -73,7 +84,8 @@ jobs: dependsOn: checkout_llvm displayName: Build llvm-tblgen linux - container: linux_x64 + # CBL-Mariner: + #container: linux_x64 pool: vmImage: ubuntu-latest @@ -84,14 +96,19 @@ jobs: steps: - template: /eng/download-checkout-llvm.yml - - script: ./build-tblgen.sh linux-x64 /crossrootfs/x64 + # CBL-Mariner: + #- script: ./build-tblgen.sh linux-x64 /crossrootfs/x64 + # displayName: Build llvm-tblgen + + # non-CBL-Mariner: + - script: ./build-tblgen.sh linux-x64 displayName: Build llvm-tblgen - publish: $(Build.SourcesDirectory)/bin/llvm-tblgen artifact: tblgen-linux displayName: Publish llvm-tblgen -################################ Cross-build coredistools for linux-x64, linux-arm, linux-arm64 in CBL-Mariner Docker container +################################ Cross-build coredistools for linux-arm, linux-arm64 (and linux-x64 on CBL-Mariner) - job: crossbuild_coredistools_linux dependsOn: @@ -106,10 +123,11 @@ jobs: strategy: matrix: - x64: - ContainerImage: linux_x64 - CrossRootfsDirectory: /crossrootfs/x64 - TargetOSArchitecture: linux-x64 + # CBL-Mariner: + #x64: + # ContainerImage: linux_x64 + # CrossRootfsDirectory: /crossrootfs/x64 + # TargetOSArchitecture: linux-x64 arm: ContainerImage: linux_arm @@ -149,6 +167,48 @@ jobs: artifact: coredistools-$(TargetOSArchitecture) displayName: Publish coredistools +################################ Cross-build coredistools for linux-x64: only non-CBL-Mariner + +- job: build_coredistools_linux + dependsOn: + - checkout_llvm + - build_tblgen_linux + displayName: Build coredistools Linux x64 + + pool: + vmImage: ubuntu-latest + + variables: + TargetOSArchitecture: linux-x64 + + workspace: + clean: all + + steps: + - template: /eng/download-checkout-llvm.yml + + - download: current + artifact: tblgen-linux + displayName: Download llvm-tblgen + + - script: | + chmod +x $(Pipeline.Workspace)/tblgen-linux/llvm-tblgen + displayName: Make llvm-tblgen executable + + - script: | + ls -lAF $(Pipeline.Workspace) + ls -lAF $(Pipeline.Workspace)/tblgen-linux + echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" + displayName: Add llvm-tblgen to the PATH + + - script: | + ./build-coredistools.sh $(TargetOSArchitecture) + displayName: Build coredistools + + - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.so + artifact: coredistools-$(TargetOSArchitecture) + displayName: Publish coredistools + ################################ Build llvm-tblgen on Mac (x64) - job: build_tblgen_macos @@ -284,6 +344,8 @@ jobs: - job: build_coredistools_nuget_packages dependsOn: - crossbuild_coredistools_linux + # only non-CBL-Mariner: + - build_coredistools_linux - build_coredistools_macos - build_coredistools_windows displayName: Build coredistools NuGet packages From 7846935e0a802e0923152868380ec0b51cdf605a Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 20 Apr 2023 20:52:54 -0700 Subject: [PATCH 13/41] Fix Linux arm/arm64 container names to match CBL-Mariner names --- coredistools.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index 96f666c6..86f206ee 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -42,9 +42,9 @@ trigger: # Use the following for legacy Ubuntu 18.04 based Arm/Arm64 cross build (x64 build is not cross) resources: containers: - - container: ubuntu-18.04-arm + - container: linux_arm image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666 - - container: ubuntu-18.04-arm64 + - container: linux_arm64 image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 variables: @@ -114,7 +114,9 @@ jobs: dependsOn: - checkout_llvm - build_tblgen_linux - displayName: Build coredistools Linux + # CBL-Mariner: + # displayName: Build coredistools Linux x64/arm/arm64 + displayName: Build coredistools Linux arm/arm64 container: $[ variables['ContainerImage'] ] From 188d60b82ef3cd1bc2adfbdf86e31bb186f93515 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 20 Apr 2023 21:51:03 -0700 Subject: [PATCH 14/41] Use clang-9 on cross-build containers This wasn't done before, so it's no clear why it's needed now. Also, it seems like the built llvm-tblgen isn't working. --- build-coredistools.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/build-coredistools.sh b/build-coredistools.sh index a94f3d58..acbefdf1 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -33,6 +33,9 @@ if [ -z "$TblGenTool" ]; then exit 1 fi +C_COMPILER=$(command -v clang) +CXX_COMPILER=$(command -v clang++) + case "$TargetOSArchitecture" in linux-arm) CMakeCrossCompiling=ON @@ -40,12 +43,20 @@ case "$TargetOSArchitecture" in LLVMHostTriple=arm-linux-gnueabihf LLVMTargetsToBuild="ARM" EnsureCrossRootfsDirectoryExists + if [ $CrossBuildUsingMariner -eq 0 ]; then + C_COMPILER=$(command -v clang-9) + CXX_COMPILER=$(command -v clang++-9) + fi ;; linux-arm64) CMakeCrossCompiling=ON LLVMHostTriple=aarch64-linux-gnu EnsureCrossRootfsDirectoryExists + if [ $CrossBuildUsingMariner -eq 0 ]; then + C_COMPILER=$(command -v clang-9) + CXX_COMPILER=$(command -v clang++-9) + fi ;; linux-x64) @@ -81,6 +92,9 @@ case "$TargetOSArchitecture" in exit 1 esac +echo "Using $C_COMPILER" +echo "Using $CXX_COMPILER" + LLVMDefaultTargetTriple=${LLVMDefaultTargetTriple:-$LLVMHostTriple} RootDirectory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -107,8 +121,8 @@ if [ -z "$CrossRootfsDirectory" ]; then -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ - -DCMAKE_C_COMPILER=$(command -v clang) \ - -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_C_COMPILER=${C_COMPILER} \ + -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ @@ -134,8 +148,8 @@ elif [ $CrossBuildUsingMariner -eq 1 ]; then -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ - -DCMAKE_C_COMPILER=$(command -v clang) \ - -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_C_COMPILER=${C_COMPILER} \ + -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5" \ @@ -159,8 +173,8 @@ else -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ - -DCMAKE_C_COMPILER=$(command -v clang) \ - -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_C_COMPILER=${C_COMPILER} \ + -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ From f1e128c797316b4b52dd82d0ff2e02ee4dad49e2 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Fri, 21 Apr 2023 09:03:43 -0700 Subject: [PATCH 15/41] For non-Mariner, use ubuntu-18.04 build pool Otherwise, we end up with clang-14 library dependencies in tblgen that don't run on our arm cross-compiler Docker images. --- build-coredistools.sh | 33 ++++++++++++++++++++------------- coredistools.yml | 10 ++++------ 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/build-coredistools.sh b/build-coredistools.sh index acbefdf1..42d6e65b 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -18,9 +18,9 @@ LLVMTargetsToBuild="AArch64;ARM;X86" # Figure out which `strip` to use. Prefer `llvm-strip` if it is available. # `llvm-strip` is available in CBL-Mariner container; `strip` is available on macOS. -StripTool=$(command -v llvm-strip) +StripTool=$(command -v strip) if [ -z "$StripTool" ]; then - StripTool=$(command -v strip) + StripTool=$(command -v llvm-strip) if [ -z "$StripTool" ]; then echo "Strip tool not found" exit 1 @@ -34,7 +34,25 @@ if [ -z "$TblGenTool" ]; then fi C_COMPILER=$(command -v clang) +if [ -z "$C_COMPILER" ]; then + C_COMPILER=$(command -v clang-9) + if [ -z "$C_COMPILER" ]; then + echo "C compiler not found" + # Keep going in case cmake can find one? + fi +fi + CXX_COMPILER=$(command -v clang++) +if [ -z "$CXX_COMPILER" ]; then + CXX_COMPILER=$(command -v clang++-9) + if [ -z "$CXX_COMPILER" ]; then + echo "C++ compiler not found" + # Keep going in case cmake can find one? + fi +fi + +echo "Using C compiler: $C_COMPILER" +echo "Using C++ compiler: $CXX_COMPILER" case "$TargetOSArchitecture" in linux-arm) @@ -43,20 +61,12 @@ case "$TargetOSArchitecture" in LLVMHostTriple=arm-linux-gnueabihf LLVMTargetsToBuild="ARM" EnsureCrossRootfsDirectoryExists - if [ $CrossBuildUsingMariner -eq 0 ]; then - C_COMPILER=$(command -v clang-9) - CXX_COMPILER=$(command -v clang++-9) - fi ;; linux-arm64) CMakeCrossCompiling=ON LLVMHostTriple=aarch64-linux-gnu EnsureCrossRootfsDirectoryExists - if [ $CrossBuildUsingMariner -eq 0 ]; then - C_COMPILER=$(command -v clang-9) - CXX_COMPILER=$(command -v clang++-9) - fi ;; linux-x64) @@ -92,9 +102,6 @@ case "$TargetOSArchitecture" in exit 1 esac -echo "Using $C_COMPILER" -echo "Using $CXX_COMPILER" - LLVMDefaultTargetTriple=${LLVMDefaultTargetTriple:-$LLVMHostTriple} RootDirectory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" diff --git a/coredistools.yml b/coredistools.yml index 86f206ee..d251a71e 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -88,7 +88,9 @@ jobs: #container: linux_x64 pool: - vmImage: ubuntu-latest + # CBL-Mariner: + # vmImage: ubuntu-latest + vmImage: ubuntu-18.04 workspace: clean: all @@ -156,8 +158,6 @@ jobs: displayName: Make llvm-tblgen executable - script: | - ls -lAF $(Pipeline.Workspace) - ls -lAF $(Pipeline.Workspace)/tblgen-linux echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" displayName: Add llvm-tblgen to the PATH @@ -178,7 +178,7 @@ jobs: displayName: Build coredistools Linux x64 pool: - vmImage: ubuntu-latest + vmImage: ubuntu-18.04 variables: TargetOSArchitecture: linux-x64 @@ -198,8 +198,6 @@ jobs: displayName: Make llvm-tblgen executable - script: | - ls -lAF $(Pipeline.Workspace) - ls -lAF $(Pipeline.Workspace)/tblgen-linux echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" displayName: Add llvm-tblgen to the PATH From 652a7aab72d27de7ba68d908e0210d597483063b Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Fri, 21 Apr 2023 11:22:51 -0700 Subject: [PATCH 16/41] AzDO pipelines no longer have ubuntu-18.04. Try ubuntu-20.04 20.04 has clang 10, not clang 9. --- build-coredistools.sh | 18 ++++++++++++------ coredistools.yml | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/build-coredistools.sh b/build-coredistools.sh index 42d6e65b..be65cd6a 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -35,19 +35,25 @@ fi C_COMPILER=$(command -v clang) if [ -z "$C_COMPILER" ]; then - C_COMPILER=$(command -v clang-9) + C_COMPILER=$(command -v clang-10) if [ -z "$C_COMPILER" ]; then - echo "C compiler not found" - # Keep going in case cmake can find one? + C_COMPILER=$(command -v clang-9) + if [ -z "$C_COMPILER" ]; then + echo "C compiler not found" + # Keep going in case cmake can find one? + fi fi fi CXX_COMPILER=$(command -v clang++) if [ -z "$CXX_COMPILER" ]; then - CXX_COMPILER=$(command -v clang++-9) + CXX_COMPILER=$(command -v clang++-10) if [ -z "$CXX_COMPILER" ]; then - echo "C++ compiler not found" - # Keep going in case cmake can find one? + CXX_COMPILER=$(command -v clang++-9) + if [ -z "$CXX_COMPILER" ]; then + echo "C++ compiler not found" + # Keep going in case cmake can find one? + fi fi fi diff --git a/coredistools.yml b/coredistools.yml index d251a71e..dcfd8a33 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -90,7 +90,7 @@ jobs: pool: # CBL-Mariner: # vmImage: ubuntu-latest - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 workspace: clean: all @@ -178,7 +178,7 @@ jobs: displayName: Build coredistools Linux x64 pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 variables: TargetOSArchitecture: linux-x64 From e83923712f5d99418ebbc01cdd7e69d3172eb6bf Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 11 May 2023 00:18:51 -0700 Subject: [PATCH 17/41] Build using CBL-Mariner and LLVM 16.0.3 --- build-coredistools.sh | 30 ++++------ build-tblgen.sh | 6 +- coredistools.yml | 134 +++++++++++++++++++++--------------------- 3 files changed, 82 insertions(+), 88 deletions(-) diff --git a/build-coredistools.sh b/build-coredistools.sh index be65cd6a..1f8fa428 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -4,7 +4,7 @@ TargetOSArchitecture=$1 CrossRootfsDirectory=$2 # Set this to 1 to build using CBL-Mariner -CrossBuildUsingMariner=0 +CrossBuildUsingMariner=1 EnsureCrossRootfsDirectoryExists () { if [ ! -d "$CrossRootfsDirectory" ]; then @@ -18,9 +18,9 @@ LLVMTargetsToBuild="AArch64;ARM;X86" # Figure out which `strip` to use. Prefer `llvm-strip` if it is available. # `llvm-strip` is available in CBL-Mariner container; `strip` is available on macOS. -StripTool=$(command -v strip) +StripTool=$(command -v llvm-strip) if [ -z "$StripTool" ]; then - StripTool=$(command -v llvm-strip) + StripTool=$(command -v strip) if [ -z "$StripTool" ]; then echo "Strip tool not found" exit 1 @@ -35,26 +35,14 @@ fi C_COMPILER=$(command -v clang) if [ -z "$C_COMPILER" ]; then - C_COMPILER=$(command -v clang-10) - if [ -z "$C_COMPILER" ]; then - C_COMPILER=$(command -v clang-9) - if [ -z "$C_COMPILER" ]; then - echo "C compiler not found" - # Keep going in case cmake can find one? - fi - fi + echo "C compiler not found" + # Keep going in case cmake can find one? fi CXX_COMPILER=$(command -v clang++) if [ -z "$CXX_COMPILER" ]; then - CXX_COMPILER=$(command -v clang++-10) - if [ -z "$CXX_COMPILER" ]; then - CXX_COMPILER=$(command -v clang++-9) - if [ -z "$CXX_COMPILER" ]; then - echo "C++ compiler not found" - # Keep going in case cmake can find one? - fi - fi + echo "C++ compiler not found" + # Keep going in case cmake can find one? fi echo "Using C compiler: $C_COMPILER" @@ -154,9 +142,11 @@ if [ -z "$CrossRootfsDirectory" ]; then elif [ $CrossBuildUsingMariner -eq 1 ]; then BUILD_FLAGS="--sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" + # [old info:] # CBL-Mariner doesn't seem to have libgcc_s.so in a standard place, so as a hack, add # -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5 # where it does exist. + # -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5" \ cmake \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ @@ -165,7 +155,7 @@ elif [ $CrossBuildUsingMariner -eq 1 ]; then -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ - -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5" \ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \ -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ diff --git a/build-tblgen.sh b/build-tblgen.sh index 5c61b7cf..c4f679aa 100755 --- a/build-tblgen.sh +++ b/build-tblgen.sh @@ -12,7 +12,7 @@ TargetOSArchitecture=$1 CrossRootfsDirectory=$2 # Set this to 1 to build using CBL-Mariner -CrossBuildUsingMariner=0 +CrossBuildUsingMariner=1 EnsureCrossRootfsDirectoryExists () { if [ ! -d "$CrossRootfsDirectory" ]; then @@ -87,9 +87,11 @@ if [ -z "$CrossRootfsDirectory" ]; then elif [ $CrossBuildUsingMariner -eq 1 ]; then BUILD_FLAGS="--sysroot=$CrossRootfsDirectory" # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" + # [old info:] # CBL-Mariner doesn't seem to have libgcc_s.so in a standard place, so as a hack, add # -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5 # where it does exist. + # -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5" \ cmake \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ @@ -99,7 +101,7 @@ elif [ $CrossBuildUsingMariner -eq 1 ]; then -DCMAKE_CXX_COMPILER=$(command -v clang++) \ -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ - -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5" \ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \ -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ diff --git a/coredistools.yml b/coredistools.yml index dcfd8a33..0439b178 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -1,4 +1,4 @@ -# Changes to build with CBL-Mariner are commented out and marked "CBL-Mariner" +# Changes to build with CBL-Mariner are marked "CBL-Mariner" pr: branches: @@ -30,27 +30,27 @@ trigger: - pack-coredistools.cmd # CBL-Mariner: -#resources: -# containers: -# - container: linux_x64 -# image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-20230414190614-8100bf7 -# - container: linux_arm -# image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-20230414190614-8100bf7 -# - container: linux_arm64 -# image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-20230414190614-8100bf7 - -# Use the following for legacy Ubuntu 18.04 based Arm/Arm64 cross build (x64 build is not cross) resources: containers: + - container: linux_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 - container: linux_arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm - container: linux_arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 + +# # Use the following for legacy Ubuntu 18.04 based Arm/Arm64 cross build (x64 build is not cross) +# resources: +# containers: +# - container: linux_arm +# image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666 +# - container: linux_arm64 +# image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 variables: LLVMRepositoryUri: https://github.com/llvm/llvm-project.git LLVMSourceBundle: llvm-project.bundle - LLVMSourceVersion: llvmorg-16.0.1 + LLVMSourceVersion: llvmorg-16.0.3 jobs: @@ -85,12 +85,13 @@ jobs: displayName: Build llvm-tblgen linux # CBL-Mariner: - #container: linux_x64 + container: linux_x64 pool: # CBL-Mariner: - # vmImage: ubuntu-latest - vmImage: ubuntu-20.04 + vmImage: ubuntu-latest + # non-CBL-Mariner: + # vmImage: ubuntu-20.04 workspace: clean: all @@ -99,12 +100,12 @@ jobs: - template: /eng/download-checkout-llvm.yml # CBL-Mariner: - #- script: ./build-tblgen.sh linux-x64 /crossrootfs/x64 - # displayName: Build llvm-tblgen + - script: ./build-tblgen.sh linux-x64 /crossrootfs/x64 + displayName: Build llvm-tblgen # non-CBL-Mariner: - - script: ./build-tblgen.sh linux-x64 - displayName: Build llvm-tblgen + #- script: ./build-tblgen.sh linux-x64 + # displayName: Build llvm-tblgen - publish: $(Build.SourcesDirectory)/bin/llvm-tblgen artifact: tblgen-linux @@ -117,8 +118,9 @@ jobs: - checkout_llvm - build_tblgen_linux # CBL-Mariner: - # displayName: Build coredistools Linux x64/arm/arm64 - displayName: Build coredistools Linux arm/arm64 + displayName: Build coredistools Linux x64/arm/arm64 + # non-CBL-Mariner: + # displayName: Build coredistools Linux arm/arm64 container: $[ variables['ContainerImage'] ] @@ -127,11 +129,11 @@ jobs: strategy: matrix: - # CBL-Mariner: - #x64: - # ContainerImage: linux_x64 - # CrossRootfsDirectory: /crossrootfs/x64 - # TargetOSArchitecture: linux-x64 + # CBL-Mariner: (x64 section) + x64: + ContainerImage: linux_x64 + CrossRootfsDirectory: /crossrootfs/x64 + TargetOSArchitecture: linux-x64 arm: ContainerImage: linux_arm @@ -171,43 +173,43 @@ jobs: ################################ Cross-build coredistools for linux-x64: only non-CBL-Mariner -- job: build_coredistools_linux - dependsOn: - - checkout_llvm - - build_tblgen_linux - displayName: Build coredistools Linux x64 - - pool: - vmImage: ubuntu-20.04 - - variables: - TargetOSArchitecture: linux-x64 - - workspace: - clean: all - - steps: - - template: /eng/download-checkout-llvm.yml - - - download: current - artifact: tblgen-linux - displayName: Download llvm-tblgen - - - script: | - chmod +x $(Pipeline.Workspace)/tblgen-linux/llvm-tblgen - displayName: Make llvm-tblgen executable - - - script: | - echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" - displayName: Add llvm-tblgen to the PATH - - - script: | - ./build-coredistools.sh $(TargetOSArchitecture) - displayName: Build coredistools - - - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.so - artifact: coredistools-$(TargetOSArchitecture) - displayName: Publish coredistools +# - job: build_coredistools_linux +# dependsOn: +# - checkout_llvm +# - build_tblgen_linux +# displayName: Build coredistools Linux x64 +# +# pool: +# vmImage: ubuntu-20.04 +# +# variables: +# TargetOSArchitecture: linux-x64 +# +# workspace: +# clean: all +# +# steps: +# - template: /eng/download-checkout-llvm.yml +# +# - download: current +# artifact: tblgen-linux +# displayName: Download llvm-tblgen +# +# - script: | +# chmod +x $(Pipeline.Workspace)/tblgen-linux/llvm-tblgen +# displayName: Make llvm-tblgen executable +# +# - script: | +# echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" +# displayName: Add llvm-tblgen to the PATH +# +# - script: | +# ./build-coredistools.sh $(TargetOSArchitecture) +# displayName: Build coredistools +# +# - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.so +# artifact: coredistools-$(TargetOSArchitecture) +# displayName: Publish coredistools ################################ Build llvm-tblgen on Mac (x64) @@ -345,7 +347,7 @@ jobs: dependsOn: - crossbuild_coredistools_linux # only non-CBL-Mariner: - - build_coredistools_linux + # - build_coredistools_linux - build_coredistools_macos - build_coredistools_windows displayName: Build coredistools NuGet packages From 36f400c6b4f8c50bd776ae85c5860fd500bfefbf Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 7 Sep 2023 10:55:38 -0700 Subject: [PATCH 18/41] Try llvmorg-16.0.6 --- coredistools.yml | 2 +- doc/building-coredistools.md | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index 0439b178..58e386d3 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -50,7 +50,7 @@ resources: variables: LLVMRepositoryUri: https://github.com/llvm/llvm-project.git LLVMSourceBundle: llvm-project.bundle - LLVMSourceVersion: llvmorg-16.0.3 + LLVMSourceVersion: llvmorg-16.0.6 jobs: diff --git a/doc/building-coredistools.md b/doc/building-coredistools.md index 844fcd3f..b8896d0f 100644 --- a/doc/building-coredistools.md +++ b/doc/building-coredistools.md @@ -10,7 +10,7 @@ cd jitutils 2. Checkout the LLVM project repository into a subdirectory named src/llvm-project: ``` -git clone --depth 1 --branch llvmorg-16.0.1 https://github.com/llvm/llvm-project.git src\llvm-project +git clone --depth 1 --branch llvmorg-16.0.6 https://github.com/llvm/llvm-project.git src\llvm-project ``` 3. Build `llvm-tblgen.exe`: @@ -63,24 +63,24 @@ cd jitutils 2. Checkout the LLVM project repository: ``` -git clone --depth 1 --branch llvmorg-16.0.1 https://github.com/llvm/llvm-project.git src/llvm-project +git clone --depth 1 --branch llvmorg-16.0.6 https://github.com/llvm/llvm-project.git src/llvm-project ``` 3. Download LLVM release from GitHub: ``` -python3 eng/download-llvm-release.py -release llvmorg-16.0.1 -os linux +python3 eng/download-llvm-release.py -release llvmorg-16.0.6 -os linux ``` 4. Locate under the current directory file `llvm-tblgen` ``` find -name llvm-tblgen -./clang+llvm-16.0.1-x86_64-linux-gnu-ubuntu-18.04/bin/llvm-tblgen +./clang+llvm-16.0.6-x86_64-linux-gnu-ubuntu-18.04/bin/llvm-tblgen ``` and add its parent directory location to the `PATH`: ``` -export PATH=$(pwd)/clang+llvm-16.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +export PATH=$(pwd)/clang+llvm-16.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ``` 5. Build `libcoredistools.so` for Linux x64: @@ -97,17 +97,15 @@ find ./artifacts -name libcoredistools.so 6. Build `libcoredistools.so` for Linux arm64 under Docker: -TODO: update these Docker images with CBL-Mariner images. - ``` -docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 -export PATH=$(pwd)/clang+llvm-16.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 +export PATH=$(pwd)/clang+llvm-16.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ./build-coredistools.sh linux-arm64 /crossrootfs/arm64 ``` 7. Build `libcoredistools.so` for Linux arm under Docker: ``` -docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666 -export PATH=$(pwd)/clang+llvm-16.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm +export PATH=$(pwd)/clang+llvm-16.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ./build-coredistools.sh linux-arm /crossrootfs/arm ``` From b685fdd17e5f281fc9417588336215da0854c31a Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 7 Sep 2023 16:04:52 -0700 Subject: [PATCH 19/41] Update CBL-Mariner docker containers to use --- coredistools.yml | 12 +++++++++--- doc/building-coredistools.md | 14 +++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index 58e386d3..33b50f73 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -31,13 +31,19 @@ trigger: # CBL-Mariner: resources: + # The base CBL-Mariner tags used to build .NET are: + # mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 + # mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm + # mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 + # However, these are based on Ubuntu 16.04 libs and do not have libstdc++ 7. + # The following containers are based on a later set of libs. containers: - container: linux_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-amd64 - container: linux_arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-arm - container: linux_arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-arm64 # # Use the following for legacy Ubuntu 18.04 based Arm/Arm64 cross build (x64 build is not cross) # resources: diff --git a/doc/building-coredistools.md b/doc/building-coredistools.md index b8896d0f..44242ceb 100644 --- a/doc/building-coredistools.md +++ b/doc/building-coredistools.md @@ -66,19 +66,15 @@ cd jitutils git clone --depth 1 --branch llvmorg-16.0.6 https://github.com/llvm/llvm-project.git src/llvm-project ``` -3. Download LLVM release from GitHub: - +3. Build `llvm-tblgen` in Docker: ``` -python3 eng/download-llvm-release.py -release llvmorg-16.0.6 -os linux +docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 +./build-tblgen.sh linux-x64 /crossrootfs/x64 ``` -4. Locate under the current directory file `llvm-tblgen` -``` -find -name llvm-tblgen -./clang+llvm-16.0.6-x86_64-linux-gnu-ubuntu-18.04/bin/llvm-tblgen -``` -and add its parent directory location to the `PATH`: +This builds llvm-tblgen and puts it in the `bin` subdirectory. +4. Add `llvm-tblgen` to the PATH: ``` export PATH=$(pwd)/clang+llvm-16.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ``` From bfc9ca8d2402e85293dde7c76b348c79399f9872 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 7 Sep 2023 22:57:26 -0700 Subject: [PATCH 20/41] More build fixes 1. Disable annoying warning message 2. Set proper CMAKE linker flags for library creation --- build-coredistools.sh | 2 +- src/coredistools/coredistools.h | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/build-coredistools.sh b/build-coredistools.sh index 1f8fa428..40422590 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -155,7 +155,7 @@ elif [ $CrossBuildUsingMariner -eq 1 ]; then -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ - -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \ + -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" \ -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ diff --git a/src/coredistools/coredistools.h b/src/coredistools/coredistools.h index f567ba3c..6937cb72 100644 --- a/src/coredistools/coredistools.h +++ b/src/coredistools/coredistools.h @@ -26,6 +26,10 @@ #define DllIface EXTERN_C __declspec(dllimport) #endif // defined(DllInterfaceExporter) #else + +// Disable "warning: default label in switch which covers all enumeration values [-Wcovered-switch-default]" +#pragma clang diagnostic ignored "-Wcovered-switch-default" + #if !defined(__cdecl) #if defined(__i386__) #define __cdecl __attribute__((cdecl)) From 7ed048ac37a98e58207ed5f11dc5b532bc7bb03f Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 7 Sep 2023 23:01:03 -0700 Subject: [PATCH 21/41] Install runtime dependency of llvm-tblgen Use `tdnf install -y ncurses-compat` to install libtinfo.so.5 dependency of llvm-tblgen. --- coredistools.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/coredistools.yml b/coredistools.yml index 33b50f73..1010627c 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -169,6 +169,10 @@ jobs: echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" displayName: Add llvm-tblgen to the PATH + - script: | + tdnf install -y ncurses-compat + displayName: Install libtinfo.so.5 dependency of llvm-tblgen + - script: | ./build-coredistools.sh $(TargetOSArchitecture) $(CrossRootfsDirectory) displayName: Build coredistools From 0a06a1c0013003f66f8d93d738bb3841bfa7758e Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Fri, 8 Sep 2023 09:53:51 -0700 Subject: [PATCH 22/41] Try using sudo --- coredistools.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coredistools.yml b/coredistools.yml index 1010627c..837b87e9 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -170,7 +170,7 @@ jobs: displayName: Add llvm-tblgen to the PATH - script: | - tdnf install -y ncurses-compat + sudo tdnf install -y ncurses-compat displayName: Install libtinfo.so.5 dependency of llvm-tblgen - script: | From 0347ce6c7175ab725aab473b504ee694a66f4217 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Fri, 8 Sep 2023 10:51:23 -0700 Subject: [PATCH 23/41] Update cmake linker flags again Even if we're building a shared library, we still need to set the cmake EXE linker flags since cmake uses it during its testing phase. --- build-coredistools.sh | 6 +----- build-tblgen.sh | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/build-coredistools.sh b/build-coredistools.sh index 40422590..9dca3afc 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -142,11 +142,6 @@ if [ -z "$CrossRootfsDirectory" ]; then elif [ $CrossBuildUsingMariner -eq 1 ]; then BUILD_FLAGS="--sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" - # [old info:] - # CBL-Mariner doesn't seem to have libgcc_s.so in a standard place, so as a hack, add - # -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5 - # where it does exist. - # -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5" \ cmake \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ @@ -155,6 +150,7 @@ elif [ $CrossBuildUsingMariner -eq 1 ]; then -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \ -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" \ -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ diff --git a/build-tblgen.sh b/build-tblgen.sh index c4f679aa..c616649a 100755 --- a/build-tblgen.sh +++ b/build-tblgen.sh @@ -87,11 +87,6 @@ if [ -z "$CrossRootfsDirectory" ]; then elif [ $CrossBuildUsingMariner -eq 1 ]; then BUILD_FLAGS="--sysroot=$CrossRootfsDirectory" # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" - # [old info:] - # CBL-Mariner doesn't seem to have libgcc_s.so in a standard place, so as a hack, add - # -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5 - # where it does exist. - # -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld -L/crossrootfs/x64/usr/lib/gcc/x86_64-linux-gnu/5" \ cmake \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ @@ -102,6 +97,7 @@ elif [ $CrossBuildUsingMariner -eq 1 ]; then -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \ + -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" \ -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ From 13f613fc4b2cf26831f53a5f5b9dc0c558d85169 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Fri, 8 Sep 2023 12:03:37 -0700 Subject: [PATCH 24/41] Explicitly specify 60 minute timeout for building coredistools job --- coredistools.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/coredistools.yml b/coredistools.yml index 837b87e9..181860db 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -120,6 +120,7 @@ jobs: ################################ Cross-build coredistools for linux-arm, linux-arm64 (and linux-x64 on CBL-Mariner) - job: crossbuild_coredistools_linux + timeoutInMinutes: 60 dependsOn: - checkout_llvm - build_tblgen_linux @@ -246,6 +247,7 @@ jobs: ################################ Build coredistools for macos-x64, macos-arm64 - job: build_coredistools_macos + timeoutInMinutes: 60 dependsOn: - checkout_llvm - build_tblgen_macos @@ -314,6 +316,7 @@ jobs: ################################ Build coredistools for win-x64, win-x86, win-arm64 - job: build_coredistools_windows + timeoutInMinutes: 60 dependsOn: - checkout_llvm - build_tblgen_windows From 4dfebad660b2bff0a6d63823ff164f156cea4a27 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Fri, 8 Sep 2023 12:04:02 -0700 Subject: [PATCH 25/41] Bump coredistools version from 1.1.0 to 1.2.0 To use this in dotnet/runtime, after the package is published, `MicrosoftNETCoreCoreDisToolsVersion` in eng/Versions.props needs to be updated. --- src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec b/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec index f6b3a69c..805c3840 100644 --- a/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec +++ b/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec @@ -1,8 +1,8 @@ - Microsoft.NETCore.CoreDisTools - 1.1.0 + Microsoft.NETCore.CoreDisTools + 1.2.0 Microsoft.NETCore Instruction-wise Disassembler Microsoft Microsoft From e91508948516a5cc0cb2323516371574245692cd Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 21 Dec 2023 09:53:37 -0800 Subject: [PATCH 26/41] Update from llvmorg-16.0.6 to llvmorg-17.0.6 --- coredistools.yml | 2 +- doc/building-coredistools.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index 181860db..ed1c3e12 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -56,7 +56,7 @@ resources: variables: LLVMRepositoryUri: https://github.com/llvm/llvm-project.git LLVMSourceBundle: llvm-project.bundle - LLVMSourceVersion: llvmorg-16.0.6 + LLVMSourceVersion: llvmorg-17.0.6 jobs: diff --git a/doc/building-coredistools.md b/doc/building-coredistools.md index 44242ceb..5ba6b8c4 100644 --- a/doc/building-coredistools.md +++ b/doc/building-coredistools.md @@ -10,7 +10,7 @@ cd jitutils 2. Checkout the LLVM project repository into a subdirectory named src/llvm-project: ``` -git clone --depth 1 --branch llvmorg-16.0.6 https://github.com/llvm/llvm-project.git src\llvm-project +git clone --depth 1 --branch llvmorg-17.0.6 https://github.com/llvm/llvm-project.git src\llvm-project ``` 3. Build `llvm-tblgen.exe`: @@ -63,7 +63,7 @@ cd jitutils 2. Checkout the LLVM project repository: ``` -git clone --depth 1 --branch llvmorg-16.0.6 https://github.com/llvm/llvm-project.git src/llvm-project +git clone --depth 1 --branch llvmorg-17.0.6 https://github.com/llvm/llvm-project.git src/llvm-project ``` 3. Build `llvm-tblgen` in Docker: @@ -76,7 +76,7 @@ This builds llvm-tblgen and puts it in the `bin` subdirectory. 4. Add `llvm-tblgen` to the PATH: ``` -export PATH=$(pwd)/clang+llvm-16.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +export PATH=$(pwd)/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ``` 5. Build `libcoredistools.so` for Linux x64: @@ -95,13 +95,13 @@ find ./artifacts -name libcoredistools.so ``` docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 -export PATH=$(pwd)/clang+llvm-16.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +export PATH=$(pwd)/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ./build-coredistools.sh linux-arm64 /crossrootfs/arm64 ``` 7. Build `libcoredistools.so` for Linux arm under Docker: ``` docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm -export PATH=$(pwd)/clang+llvm-16.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +export PATH=$(pwd)/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH ./build-coredistools.sh linux-arm /crossrootfs/arm ``` From 78556fce652e7dc574cd02bc54de820481827b94 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 21 Dec 2023 14:52:20 -0800 Subject: [PATCH 27/41] Fix build --- build-coredistools.cmd | 6 ++++-- doc/building-coredistools.md | 16 +++++----------- src/coredistools/coredistools.cpp | 2 -- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/build-coredistools.cmd b/build-coredistools.cmd index ffe11e5a..5f47e83f 100644 --- a/build-coredistools.cmd +++ b/build-coredistools.cmd @@ -52,6 +52,9 @@ if not exist "%BinariesDirectory%" ( ) pushd "%BinariesDirectory%" + +@REM For Debug builds, use: +@REM -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebug cmake.exe ^ -G "Visual Studio 17 2022" ^ @@ -65,8 +68,7 @@ cmake.exe ^ -DLLVM_TABLEGEN="%LLVMTableGen%" ^ -DLLVM_TARGETS_TO_BUILD=%LLVMTargetsToBuild% ^ -DLLVM_TOOL_COREDISTOOLS_BUILD=ON ^ - -DLLVM_USE_CRT_DEBUG=MTd ^ - -DLLVM_USE_CRT_RELEASE=MT ^ + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded ^ "%SourcesDirectory%\llvm-project\llvm" popd diff --git a/doc/building-coredistools.md b/doc/building-coredistools.md index 5ba6b8c4..fcd64745 100644 --- a/doc/building-coredistools.md +++ b/doc/building-coredistools.md @@ -28,29 +28,23 @@ set "PATH=%cd%\bin;%PATH%" This puts the just built lldb-tblgen.exe on the `PATH`. 5. Build `coredistools.dll` for a combination of target OS and architecture. - -For example, the following command will result in `coredistools.dll` binary that can be run on Windows x64: +Build Windows x64, Windows x86, and Windows ARM64 binaries: ``` build-coredistools.cmd win-x64 +build-coredistools.cmd win-x86 +build-coredistools.cmd win-arm64 ``` -The file will be copied to subdirectory `artifacts` after the command finishes: +The file will be copied to subdirectory `artifacts` after the command finishes. E.g., for win-x64: ``` dir /A:-D /B /S artifacts\win-x64 F:\echesako\git\jitutils\artifacts\win-x64\bin\coredistools.dll F:\echesako\git\jitutils\artifacts\win-x64\lib\coredistools.lib ``` -6. Build Windows x86, Windows ARM and Windows ARM64 binaries: -``` -build-coredistools.cmd win-x86 -build-coredistools.cmd win-arm -build-coredistools.cmd win-arm64 -``` - ### Building Debug binaries -The `build-coredistools.cmd` script is set up to build a Release build. To create a Debug build with a PDB file, +The `build-coredistools.cmd` script is set up to build a Release build. To create a Debug build with a PDB file for debugging, change the `--config Release` line to `--config Debug`. ## Building on Linux / Mac diff --git a/src/coredistools/coredistools.cpp b/src/coredistools/coredistools.cpp index 9648c28f..792d1447 100644 --- a/src/coredistools/coredistools.cpp +++ b/src/coredistools/coredistools.cpp @@ -11,8 +11,6 @@ /// //===----------------------------------------------------------------------===// -#include "llvm/ADT/Optional.h" -#include "llvm/ADT/Triple.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDisassembler/MCDisassembler.h" From 1d3432a075e47a5f904589465f733cce0047f01d Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 21 Dec 2023 15:37:35 -0800 Subject: [PATCH 28/41] Fix build warning --- src/coredistools/coredistools.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coredistools/coredistools.cpp b/src/coredistools/coredistools.cpp index 792d1447..29595771 100644 --- a/src/coredistools/coredistools.cpp +++ b/src/coredistools/coredistools.cpp @@ -20,12 +20,12 @@ #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/TargetParser/Host.h" #include "llvm/Support/Format.h" -#include "llvm/Support/Host.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/SourceMgr.h" -#include "llvm/MC/TargetRegistry.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/DataTypes.h" From 13316c5e2cb86869864c17cbff733c7aa844c3be Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 21 Dec 2023 15:59:58 -0800 Subject: [PATCH 29/41] More fixes Output more diagnostics on Linux build-coredistools build Update documentation for building Linux --- coredistools.yml | 11 ++++++++++- doc/building-coredistools.md | 22 ++++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index ed1c3e12..3810eec2 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -106,7 +106,13 @@ jobs: - template: /eng/download-checkout-llvm.yml # CBL-Mariner: - - script: ./build-tblgen.sh linux-x64 /crossrootfs/x64 + + - script: | + sudo tdnf install -y ncurses-compat + displayName: Install libtinfo.so.5 dependency of llvm-tblgen + + - script: | + ./build-tblgen.sh linux-x64 /crossrootfs/x64 displayName: Build llvm-tblgen # non-CBL-Mariner: @@ -168,6 +174,9 @@ jobs: - script: | echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" + echo "$PATH" + echo "$(Pipeline.Workspace)/tblgen-linux" + ls -l "$(Pipeline.Workspace)/tblgen-linux" displayName: Add llvm-tblgen to the PATH - script: | diff --git a/doc/building-coredistools.md b/doc/building-coredistools.md index fcd64745..9651e5f4 100644 --- a/doc/building-coredistools.md +++ b/doc/building-coredistools.md @@ -61,8 +61,12 @@ git clone --depth 1 --branch llvmorg-17.0.6 https://github.com/llvm/llvm-project ``` 3. Build `llvm-tblgen` in Docker: +You need to install the `ncurses-compat` package because the Mariner container we use doesn't have libtinfo.so.5, which the built +llvm-tblgen needs to be able to run. (Note that we build in the Mariner container, but we also run some built binaries, namely llvm-tblgen, +as part of the build process.) ``` -docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 +docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-amd64 +sudo tdnf install -y ncurses-compat ./build-tblgen.sh linux-x64 /crossrootfs/x64 ``` @@ -70,12 +74,12 @@ This builds llvm-tblgen and puts it in the `bin` subdirectory. 4. Add `llvm-tblgen` to the PATH: ``` -export PATH=$(pwd)/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +export PATH=$(pwd)/bin:$PATH ``` -5. Build `libcoredistools.so` for Linux x64: +5. Build `libcoredistools.so` for Linux x64 (in the same Docker container): ``` -./build-coredistools.sh linux-x64 +./build-coredistools.sh linux-x64 /crossrootfs/x64 ``` The file will be copied to subdirectory `artifacts` after the command finishes: @@ -88,14 +92,16 @@ find ./artifacts -name libcoredistools.so 6. Build `libcoredistools.so` for Linux arm64 under Docker: ``` -docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 -export PATH=$(pwd)/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-arm64 +sudo tdnf install -y ncurses-compat +export PATH=$(pwd)/bin:$PATH ./build-coredistools.sh linux-arm64 /crossrootfs/arm64 ``` 7. Build `libcoredistools.so` for Linux arm under Docker: ``` -docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm -export PATH=$(pwd)/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-arm +sudo tdnf install -y ncurses-compat +export PATH=$(pwd)/bin:$PATH ./build-coredistools.sh linux-arm /crossrootfs/arm ``` From 3f0865a720ccf67e6a2bc3dc445192cf8b922eff Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 21 Dec 2023 16:32:06 -0800 Subject: [PATCH 30/41] Fix task display name for "matrix" Linux build The matrix appends the architecture to the displayName automatically. Remove some debugging output. --- coredistools.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index 3810eec2..8958c072 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -131,9 +131,9 @@ jobs: - checkout_llvm - build_tblgen_linux # CBL-Mariner: - displayName: Build coredistools Linux x64/arm/arm64 + displayName: Build coredistools Linux # non-CBL-Mariner: - # displayName: Build coredistools Linux arm/arm64 + # displayName: Build coredistools Linux container: $[ variables['ContainerImage'] ] @@ -142,7 +142,7 @@ jobs: strategy: matrix: - # CBL-Mariner: (x64 section) + # CBL-Mariner (x64 section only): x64: ContainerImage: linux_x64 CrossRootfsDirectory: /crossrootfs/x64 @@ -174,9 +174,6 @@ jobs: - script: | echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" - echo "$PATH" - echo "$(Pipeline.Workspace)/tblgen-linux" - ls -l "$(Pipeline.Workspace)/tblgen-linux" displayName: Add llvm-tblgen to the PATH - script: | From e67a98e17b4bf6bcd40dcf4cda075713693f92e2 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 21 Dec 2023 17:03:32 -0800 Subject: [PATCH 31/41] Add task to install nuget.exe --- coredistools.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/coredistools.yml b/coredistools.yml index 8958c072..b6a44131 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -381,6 +381,10 @@ jobs: - download: current displayName: Download artifacts + # Install nuget.exe if it's not already there. + - task: NuGetToolInstaller@1 + displayName: Install nuget.exe + - script: | for %%I in (linux-arm linux-arm64 linux-x64 osx-arm64 osx-x64 win-arm64 win-x64 win-x86) do ( mkdir "$(Build.BinariesDirectory)\%%I" From 72f418857f5de103164fa4636d052cca7672717a Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 21 Dec 2023 17:28:10 -0800 Subject: [PATCH 32/41] Bump up timeout In desperation --- coredistools.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/coredistools.yml b/coredistools.yml index b6a44131..cfb69a92 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -126,7 +126,7 @@ jobs: ################################ Cross-build coredistools for linux-arm, linux-arm64 (and linux-x64 on CBL-Mariner) - job: crossbuild_coredistools_linux - timeoutInMinutes: 60 + timeoutInMinutes: 120 dependsOn: - checkout_llvm - build_tblgen_linux @@ -191,6 +191,7 @@ jobs: ################################ Cross-build coredistools for linux-x64: only non-CBL-Mariner # - job: build_coredistools_linux +# timeoutInMinutes: 120 # dependsOn: # - checkout_llvm # - build_tblgen_linux @@ -253,7 +254,7 @@ jobs: ################################ Build coredistools for macos-x64, macos-arm64 - job: build_coredistools_macos - timeoutInMinutes: 60 + timeoutInMinutes: 120 dependsOn: - checkout_llvm - build_tblgen_macos @@ -322,7 +323,7 @@ jobs: ################################ Build coredistools for win-x64, win-x86, win-arm64 - job: build_coredistools_windows - timeoutInMinutes: 60 + timeoutInMinutes: 120 dependsOn: - checkout_llvm - build_tblgen_windows From e093dc1b1519ac9c6fd488a23ba341d05f42019f Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 21 Dec 2023 17:43:17 -0800 Subject: [PATCH 33/41] Temporary (?): use cmake verbose switch to build --- build-coredistools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-coredistools.sh b/build-coredistools.sh index 9dca3afc..be8c02ba 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -140,7 +140,7 @@ if [ -z "$CrossRootfsDirectory" ]; then -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ $SourcesDirectory/llvm-project/llvm elif [ $CrossBuildUsingMariner -eq 1 ]; then - BUILD_FLAGS="--sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" + BUILD_FLAGS="-v --sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" cmake \ -G "Unix Makefiles" \ From 46f7d8933354a728921faba355d6fe29b79cbd76 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 21 Dec 2023 18:37:52 -0800 Subject: [PATCH 34/41] Remove -v build flag. Reduce timeout to 60 minutes. --- build-coredistools.sh | 2 +- coredistools.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build-coredistools.sh b/build-coredistools.sh index be8c02ba..9dca3afc 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -140,7 +140,7 @@ if [ -z "$CrossRootfsDirectory" ]; then -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ $SourcesDirectory/llvm-project/llvm elif [ $CrossBuildUsingMariner -eq 1 ]; then - BUILD_FLAGS="-v --sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" + BUILD_FLAGS="--sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" cmake \ -G "Unix Makefiles" \ diff --git a/coredistools.yml b/coredistools.yml index cfb69a92..9e6e69b6 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -126,7 +126,7 @@ jobs: ################################ Cross-build coredistools for linux-arm, linux-arm64 (and linux-x64 on CBL-Mariner) - job: crossbuild_coredistools_linux - timeoutInMinutes: 120 + timeoutInMinutes: 60 dependsOn: - checkout_llvm - build_tblgen_linux @@ -191,7 +191,7 @@ jobs: ################################ Cross-build coredistools for linux-x64: only non-CBL-Mariner # - job: build_coredistools_linux -# timeoutInMinutes: 120 +# timeoutInMinutes: 60 # dependsOn: # - checkout_llvm # - build_tblgen_linux @@ -254,7 +254,7 @@ jobs: ################################ Build coredistools for macos-x64, macos-arm64 - job: build_coredistools_macos - timeoutInMinutes: 120 + timeoutInMinutes: 60 dependsOn: - checkout_llvm - build_tblgen_macos @@ -323,7 +323,7 @@ jobs: ################################ Build coredistools for win-x64, win-x86, win-arm64 - job: build_coredistools_windows - timeoutInMinutes: 120 + timeoutInMinutes: 60 dependsOn: - checkout_llvm - build_tblgen_windows From 0475ccd65fe3b9aa54bb15790bf37c3ad2e54bf8 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Fri, 22 Dec 2023 23:55:04 -0800 Subject: [PATCH 35/41] Various updates 1. Update header file to have type definitions for all DLL exported functions. 2. Remove unnecessary x86 instruction prefix handling (it was working around a bug that is apparently fixed). 3. Remove coredistools special handling of movw/movt. Instead, add a hopefully more general mechanism where cordistools will optionally first call a "munger" function on constants. This callback in the superpmi NearDiffer for arm32 will decode the movw/movt and treat the constructed constant as being generated by one instruction. This can also be used by arm64 mov/movk/movk/movk, although we currently don't have a need for that. --- build-coredistools.cmd | 36 ++++- src/coredistools/coredistools.cpp | 212 +++++++------------------- src/coredistools/coredistools.exports | 5 +- src/coredistools/coredistools.h | 79 ++++++++-- 4 files changed, 155 insertions(+), 177 deletions(-) diff --git a/build-coredistools.cmd b/build-coredistools.cmd index 5f47e83f..04da1b47 100644 --- a/build-coredistools.cmd +++ b/build-coredistools.cmd @@ -20,6 +20,18 @@ if /i "%TargetOSArchitecture%" == "win-arm64" ( exit /b 1 ) +set BuildFlavor=%2 +if "%BuildFlavor%"=="" set BuildFlavor=Release + +if /i "%BuildFlavor%" == "Release" ( + @REM ok +) else if /i "%BuildFlavor%" == "Debug" ( + @REM ok +) else ( + echo ERROR: Unknown build flavor: %BuildFlavor% + exit /b 1 +) + if not defined LLVMDefaultTargetTriple ( set LLVMDefaultTargetTriple=%LLVMHostTriple% ) @@ -38,11 +50,19 @@ if %ERRORLEVEL% neq 0 ( where /q llvm-tblgen.exe -if %ERRORLEVEL% neq 0 ( +if %ERRORLEVEL% equ 0 goto found_llvm_tblgen + +@REM We expect it to be in the `bin` directory, so add that to the PATH if it's there. +if not exist %RootDirectory%bin\llvm-tblgen.exe ( echo ERROR: llvm-tblgen.exe is not found in the PATH exit /b 1 ) +echo Found llvm-tblgen.exe in %RootDirectory%bin; adding that directory to PATH. +set PATH=%RootDirectory%bin;%PATH% + +:found_llvm_tblgen + for /f %%I in ('where llvm-tblgen.exe') do ( set LLVMTableGen=%%~I ) @@ -53,13 +73,21 @@ if not exist "%BinariesDirectory%" ( pushd "%BinariesDirectory%" -@REM For Debug builds, use: +@REM To use the Debug CRT, use: @REM -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebug +@REM +@REM To build a Debug version (asserts, debug info): +@REM -DCMAKE_BUILD_TYPE=Debug +@REM To build a Release version (no asserts, no debug info): +@REM -DCMAKE_BUILD_TYPE=Release +@REM +@REM Misc. LLVM CMake documentation: https://llvm.org/docs/CMake.html cmake.exe ^ -G "Visual Studio 17 2022" ^ -A %GeneratorPlatform% ^ -DCMAKE_INSTALL_PREFIX="%StagingDirectory%" ^ + -DCMAKE_BUILD_TYPE=%BuildFlavor% ^ -DLLVM_DEFAULT_TARGET_TRIPLE=%LLVMDefaultTargetTriple% ^ -DLLVM_EXTERNAL_PROJECTS=coredistools ^ -DLLVM_EXTERNAL_COREDISTOOLS_SOURCE_DIR="%SourcesDirectory%\coredistools" ^ @@ -75,10 +103,12 @@ popd if %ERRORLEVEL% neq 0 goto :CMakeNonZeroExitStatus +@REM Use `--config Release` for release build, `--config Debug` for debug build + cmake.exe ^ --build "%BinariesDirectory%" ^ --target coredistools ^ - --config Release + --config %BuildFlavor% if %ERRORLEVEL% neq 0 goto :CMakeNonZeroExitStatus diff --git a/src/coredistools/coredistools.cpp b/src/coredistools/coredistools.cpp index 29595771..08867f65 100644 --- a/src/coredistools/coredistools.cpp +++ b/src/coredistools/coredistools.cpp @@ -1,4 +1,4 @@ -//===-------- coredistools.cpp - Dissassembly tools for CoreClr -----------===// +//===-------- coredistools.cpp - Disassembly tools for CoreClr ------------===// // // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. @@ -49,23 +49,26 @@ class BlockInfo { bool isEmpty() const { return BlockSize == 0; } - // A pointer to the code block to diassemble. + // A pointer to the code block to disassemble. const uint8_t *Ptr; + // The size of the code block to compare. uint64_t BlockSize; + // The original base address of the code block. uintptr_t Addr; + // An identifying string, debug output only const char *Name; }; -// A block iterator reoresents a code-point within a code block. +// A block iterator represents a code-point within a code block. // It represents an instruction within the block, and can move // forward to subsequent instructions via the advance() method. // The iterator can be in two modes: // Not-Decoded: Before DecodeInstruction() is called at this // code point -// Decoded: After the current instrction is Decoded. In this state, +// Decoded: After the current instruction is Decoded. In this state, // Inst is a valid MCInst and InstrSize is an actual non-zero // length of the current instruction. class BlockIterator : public BlockInfo { @@ -182,8 +185,6 @@ struct CorDisasm { protected: enum TargetArch TheTargetArch; const PrintControl *Print; - bool isThumb2MoveImmediateOpcode(unsigned int opcode) const; - bool isThumb2MoveTopOpcode(unsigned int opcode) const; private: bool setTarget(); @@ -199,30 +200,15 @@ struct CorDisasm { unique_ptr Ctx; unique_ptr Disassembler; unique_ptr IP; - - // LLVM's MCInst does not expose Opcode enumerations by design. - // The following enumeration is a hack to use X86 opcode numbers, - // until bug 7709 is fixed. - struct OpcodeMap { - const char *Name; - uint8_t MachineOpcode; - }; - - static const int X86NumPrefixes = 19; - static const OpcodeMap X86Prefix[X86NumPrefixes]; - - // The following constants is a workaround and the opcode numbers - // were copied from TableGen-erated ${LLVM_BINARY_DIR}/lib/Target/ARM/ARMGenInstrInfo.inc - static const unsigned int Thumb2MoveImmediateOpcode = 4069; // Corresponds to t2MOVi16 - static const unsigned int Thumb2MoveTopOpcode = 4067; // Correspond to t2MOVTi16 }; struct CorAsmDiff : public CorDisasm { public: CorAsmDiff(enum TargetArch Target, const PrintControl *PControl = &DefaultPrintControl, - const OffsetComparator Comp = DefaultEqualityComparator) - : CorDisasm(Target, PControl), Comparator(Comp) {} + const OffsetComparator Comp = DefaultEqualityComparator, + const OffsetMunger Munge = nullptr) + : CorDisasm(Target, PControl), Comparator(Comp), Munger(Munge) {} bool nearDiff(const BlockInfo &LeftBlock, const BlockInfo &RightBlock, const void *UserData) const; @@ -231,35 +217,10 @@ struct CorAsmDiff : public CorDisasm { bool fail(const char *Mesg, const BlockIterator &Left, const BlockIterator &Right) const; - bool tryDecodeThumb2MoveImm32(const BlockIterator& Iter, unsigned int& Reg, unsigned int& Imm32) const; - OffsetComparator Comparator; + OffsetMunger Munger; }; -// clang-format off -CorDisasm::OpcodeMap const CorDisasm::X86Prefix[CorDisasm::X86NumPrefixes] = { - { "LOCK", 0xF0 }, - { "REPNE/XACQUIRE", 0xF2 }, // Both the (TSX/normal) instrs - { "REP/XRELEASE", 0xF3 }, // have the same byte encoding - { "OP_OVR", 0x66 }, - { "CS_OVR", 0x2E }, - { "DS_OVR", 0x3E }, - { "ES_OVR", 0x26 }, - { "FS_OVR", 0x64 }, - { "GS_OVR", 0x65 }, - { "SS_OVR", 0x36 }, - { "ADDR_OVR", 0x67 }, - { "REX64W", 0x48 }, - { "REX64WB", 0x49 }, - { "REX64WX", 0x4A }, - { "REX64WXB", 0x4B }, - { "REX64WR", 0x4C }, - { "REX64WRB", 0x4D }, - { "REX64WRX", 0x4E }, - { "REX64WRXB", 0x4F } -}; -// clang-format on - bool CorDisasm::setTarget() { // Figure out the target triple. @@ -363,7 +324,7 @@ bool CorDisasm::init() { string FeaturesStr; // No additional target specific attributes. if (TheTargetArch == Target_Arm64) { - Mcpu = "cortex-a76"; + Mcpu = "neoverse-n2"; } STI.reset(TheTarget->createMCSubtargetInfo(TargetTriple, Mcpu, FeaturesStr)); @@ -434,46 +395,17 @@ bool CorDisasm::decodeInstruction(BlockIterator &BIter, bool MayFail) const { uint64_t CorDisasm::disasmInstruction(BlockIterator &BIter, bool DumpAsm) const { - uint64_t TotalSize = 0; - bool ContinueDisasm; - - // On X86, LLVM disassembler does not handle instruction prefixes - // correctly -- please see LLVM bug 7709. - // The disassembler reports instruction prefixes separate from the - // actual instruction. In order to work-around this problem, we - // continue decoding past the prefix bytes. - - do { - - if (!decodeInstruction(BIter)) { - return 0; - } - - uint64_t Size = BIter.InstrSize; - TotalSize += Size; + if (!decodeInstruction(BIter)) { + return 0; + } - if (DumpAsm) { - dumpInstruction(BIter); - } + uint64_t Size = BIter.InstrSize; - ContinueDisasm = false; - if ((TheTargetArch == Target_X86) || (TheTargetArch == Target_X64)) { - - // Check if the decoded instruction is a prefix byte, and if so, - // continue decoding. - if (Size == 1) { - for (uint8_t Pfx = 0; Pfx < X86NumPrefixes; Pfx++) { - if (BIter.Ptr[0] == X86Prefix[Pfx].MachineOpcode) { - ContinueDisasm = true; - BIter.advance(); - break; - } - } - } - } - } while (ContinueDisasm); + if (DumpAsm) { + dumpInstruction(BIter); + } - return TotalSize; + return Size; } void CorDisasm::dumpInstruction(const BlockIterator &BIter) const { @@ -529,57 +461,6 @@ void CorDisasm::dumpBlock(const BlockInfo &Block) const { Print->Dump("-----------------------------------------------"); } -bool CorDisasm::isThumb2MoveImmediateOpcode(unsigned int Opcode) const { - return (Opcode == Thumb2MoveImmediateOpcode); -} - -bool CorDisasm::isThumb2MoveTopOpcode(unsigned int Opcode) const { - return (Opcode == Thumb2MoveTopOpcode); -} - -bool CorAsmDiff::tryDecodeThumb2MoveImm32(const BlockIterator& Curr, unsigned int& Reg, unsigned int& Imm32) const { - assert(Curr.isDecoded()); - - if (!isThumb2MoveImmediateOpcode(Curr.Inst.getOpcode())) { - return false; - } - - BlockIterator Next = Curr; - - Next.advance(); - - if (Next.isEmpty()) { - return false; - } - - decodeInstruction(Next); - - if (!Next.isDecoded()) { - return false; - } - - if (!isThumb2MoveTopOpcode(Next.Inst.getOpcode())) { - return false; - } - - const MCInst& MovImm = Curr.Inst; - const MCInst& MovTop = Next.Inst; - - if (MovImm.getOperand(0).getReg() != MovTop.getOperand(0).getReg()) { - return false; - } - - Reg = MovImm.getOperand(0).getReg(); - - const unsigned Lo16 = (unsigned int)MovImm.getOperand(1).getImm(); - const unsigned Hi16 = (unsigned int)MovTop.getOperand(2).getImm(); - - // Reconstruct 32-bit value that is loaded using movw/movt instruction pair. - Imm32 = Lo16 | (Hi16 << 16); - - return true; -} - // Compares two code sections for syntactic equality. This is the core of the // asm diffing logic. // @@ -631,31 +512,37 @@ bool CorAsmDiff::nearDiff(const BlockInfo &LeftBlock, return fail("Instruction Size Mismatch", Left, Right); } - if (TheTargetArch == Target_Thumb) { - unsigned int RegL = 0; - unsigned int RegR = 0; - - unsigned int Imm32L = 0; - unsigned int Imm32R = 0; + if (Munger != nullptr) + { + // Need to call the munger first + + uint64_t ImmL = 0; + uint64_t ImmR = 0; + uint32_t SkipL = 0; + uint32_t SkipR = 0; + + if (Munger(UserData, Left.BlockOffset(), Left.InstrSize, &ImmL, &ImmR, &SkipL, &SkipR)) + { + const bool constMatches = ((ImmL == ImmR) || Comparator(UserData, Left.BlockOffset(), Left.InstrSize, ImmL, ImmR)); + if (!constMatches) + { + return fail("Munged Immediate Operand Value Mismatch", Left, Right); + } - // On Thumb2 movw/movt instruction pair can be used to load a 32-bit value to a register. - // If this is the case, we should treat such instruction pair as **one** pseudo-instruction and try decoding them together. - if (tryDecodeThumb2MoveImm32(Left, RegL, Imm32L) && tryDecodeThumb2MoveImm32(Right, RegR, Imm32R)) { - if ((RegL == RegR) && ((Imm32L == Imm32R) || Comparator(UserData, Left.BlockOffset(), Left.InstrSize, Imm32L, Imm32R))) { - // When movw/movt instructions pairs load the same or "equivalent" 32-bit values - // to the same register advance both iterators to the positions after movt. + Left.advance(); + Right.advance(); - Left.advance(); + for (uint32_t i = 0; i < SkipL; i++) { decodeInstruction(Left); Left.advance(); + } - Right.advance(); + for (uint32_t i = 0; i < SkipR; i++) { decodeInstruction(Right); Right.advance(); - - continue; } - // Otherwise, do nothing and allow the comparison below to fail. + + continue; } } @@ -774,6 +661,8 @@ DllIface CorDisasm *NewDisasm(enum TargetArch Target, return nullptr; } +DllIface void FinishDisasm(const CorDisasm *Disasm) { delete Disasm; } + DllIface CorDisasm *InitBufferedDiffer(enum TargetArch Target, const OffsetComparator Comparator) { return NewDiffer(Target, &BufferedPrintControl, Comparator); @@ -782,7 +671,14 @@ DllIface CorDisasm *InitBufferedDiffer(enum TargetArch Target, DllIface CorAsmDiff *NewDiffer(enum TargetArch Target, const PrintControl *PControl, const OffsetComparator Comparator) { - CorAsmDiff *AsmDiff = new CorAsmDiff(Target, PControl, Comparator); + return NewDiffer2(Target, PControl, Comparator, nullptr); +} + +DllIface CorAsmDiff *NewDiffer2(enum TargetArch Target, + const PrintControl *PControl, + const OffsetComparator Comparator, + const OffsetMunger Munger) { + CorAsmDiff *AsmDiff = new CorAsmDiff(Target, PControl, Comparator, Munger); if (AsmDiff->init()) { return AsmDiff; @@ -792,8 +688,6 @@ DllIface CorAsmDiff *NewDiffer(enum TargetArch Target, return nullptr; } -DllIface void FinishDisasm(const CorDisasm *Disasm) { delete Disasm; } - DllIface void FinishDiff(const CorAsmDiff *AsmDiff) { delete AsmDiff; } DllIface size_t DisasmInstruction(const CorDisasm *Disasm, diff --git a/src/coredistools/coredistools.exports b/src/coredistools/coredistools.exports index 6506557c..9579e591 100644 --- a/src/coredistools/coredistools.exports +++ b/src/coredistools/coredistools.exports @@ -2,11 +2,12 @@ InitDisasm InitBufferedDisasm NewDisasm FinishDisasm -DisasmInstruction -DumpInstruction InitBufferedDiffer NewDiffer +NewDiffer2 FinishDiff +DisasmInstruction +DumpInstruction NearDiffCodeBlocks DumpCodeBlock DumpDiffBlocks diff --git a/src/coredistools/coredistools.h b/src/coredistools/coredistools.h index 6937cb72..5016e836 100644 --- a/src/coredistools/coredistools.h +++ b/src/coredistools/coredistools.h @@ -3,9 +3,9 @@ //===--------- coredistools.h - Dissassembly tools for CoreClr ------------===// // -// Core Disassembly Tools API Version 1.0.1-prerelease +// Core Disassembly Tools API Version 1.2.0 // Disassembly tools required by CoreCLR for utilities like -// GCStress and SuperPMI +// GCStress, SuperPMI, and R2RDump. //===----------------------------------------------------------------------===// #if !defined(_COREDISTOOLS_H_) @@ -65,7 +65,7 @@ struct PrintControl { // The type of a custom function provided by the user to determine // if two offsets are considered equivalent wrt diffing code blocks. // Offset1 and Offset2 are the two offsets to be compared. -// BlockOffset is the offest of the instructions (that contain Offset1 +// BlockOffset is the offset of the instructions (that contain Offset1 // and Offset2) from the beginning of their respective code blocks. // InstructionLength is the length of the current instruction being // compared for equivalency. @@ -73,6 +73,20 @@ typedef bool(__cdecl *OffsetComparator)(const void *UserData, size_t BlockOffset size_t InstructionLength, uint64_t Offset1, uint64_t Offset2); +// If an OffsetMunger function is defined, it is called before the OffsetComparator. +// If it returns `true` then: +// 1. the instructions are considered equivalent +// 2. the offsets have been decoded and "munged" (changed), and +// *Offset1 and *Offset2 are set to the values to use. +// 3. *SkipInstructions1 instructions in code stream 1 are skipped +// 4. *SkipInstructions2 instructions in code stream 2 are skipped +// +// This is typically used on arm32 to treat "movw/movt" as a single instruction +// generating a single constant. Similarly, for arm64 mov/movk/movk/movk sequences. +typedef bool(__cdecl *OffsetMunger)(const void *UserData, size_t BlockOffset, + size_t InstructionLength, uint64_t* Offset1, uint64_t* Offset2, + uint32_t* SkipInstructions1, uint32_t* SkipInstructions2); + // The Export/Import definitions for CoreDistools library are defined below. // A typedef for each interface function's type is defined in order to aid // the importer. @@ -81,6 +95,10 @@ typedef bool(__cdecl *OffsetComparator)(const void *UserData, size_t BlockOffset typedef CorDisasm * __cdecl InitDisasm_t(enum TargetArch Target); DllIface InitDisasm_t InitDisasm; +// Initialize the disassembler, using buffered print controls +typedef CorDisasm * __cdecl InitBufferedDisasm_t(enum TargetArch Target); +DllIface InitBufferedDisasm_t InitBufferedDisasm; + // Initialize the disassembler using custom print controls typedef CorDisasm * __cdecl NewDisasm_t(enum TargetArch Target, const PrintControl *PControl); @@ -90,6 +108,28 @@ DllIface NewDisasm_t NewDisasm; typedef void __cdecl FinishDisasm_t(const CorDisasm *Disasm); DllIface FinishDisasm_t FinishDisasm; +// Initialize a code differ using buffered output. +typedef CorDisasm * __cdecl InitBufferedDiffer_t(enum TargetArch Target, + const OffsetComparator Comparator); +DllIface InitBufferedDiffer_t InitBufferedDiffer; + +// Initialize the Code Differ +typedef CorAsmDiff * __cdecl NewDiffer_t(enum TargetArch Target, + const PrintControl *PControl, + const OffsetComparator Comparator); +DllIface NewDiffer_t NewDiffer; + +// Initialize the Code Differ, with an offset munger. +typedef CorAsmDiff * __cdecl NewDiffer2_t(enum TargetArch Target, + const PrintControl *PControl, + const OffsetComparator Comparator, + const OffsetMunger Munger); +DllIface NewDiffer2_t NewDiffer2; + +// Delete the Code Differ +typedef void __cdecl FinishDiff_t(const CorAsmDiff *AsmDiff); +DllIface FinishDiff_t FinishDiff; + // DisasmInstruction -- Disassemble one instruction // Arguments: // Disasm -- The Disassembler @@ -105,22 +145,27 @@ typedef size_t __cdecl DisasmInstruction_t(const CorDisasm *Disasm, const uint8_t *Bytes, size_t Maxlength); DllIface DisasmInstruction_t DisasmInstruction; -// Initialize the Code Differ -typedef CorAsmDiff * __cdecl NewDiffer_t(enum TargetArch Target, - const PrintControl *PControl, - const OffsetComparator Comparator); -DllIface NewDiffer_t NewDiffer; - -// Delete the Code Differ -typedef void __cdecl FinishDiff_t(const CorAsmDiff *AsmDiff); -DllIface FinishDiff_t FinishDiff; +// DumpInstruction -- Disassemble one instruction and output it +// Arguments: +// Disasm -- The Disassembler +// Address -- The address at which the bytes of the instruction +// are intended to execute +// Bytes -- Pointer to the actual bytes which need to be disassembled +// MaxLength -- Number of bytes available in Bytes buffer +// Returns: +// -- The Size of the disassembled instruction +// -- Zero on failure +typedef size_t __cdecl DumpInstruction_t(const CorDisasm *Disasm, + const uint8_t *Address, const uint8_t *Bytes, + size_t Maxlength); +DllIface DumpInstruction_t DumpInstruction; // NearDiffCodeBlocks -- Compare two code blocks for semantic // equivalence // Arguments: // AsmDiff -- The Asm-differ // UserData -- Any data the user wishes to pass through into -// the OffsetComparator +// the OffsetComparator/OffsetMunger // Address1 -- Address at which first block will execute // Bytes1 -- Pointer to the actual bytes of the first block // Size1 -- The size of the first block @@ -150,4 +195,12 @@ typedef void __cdecl DumpDiffBlocks_t(const CorAsmDiff *AsmDiff, const uint8_t *Bytes2, size_t Size2); DllIface DumpDiffBlocks_t DumpDiffBlocks; +// Get a pointer to the buffered output buffer. +typedef const char* __cdecl GetOutputBuffer_t(); +DllIface GetOutputBuffer_t GetOutputBuffer; + +// Clear the buffered output buffer. +typedef void __cdecl ClearOutputBuffer_t(); +DllIface ClearOutputBuffer_t ClearOutputBuffer; + #endif // !defined(_COREDISTOOLS_H_) From 15485b03558e82cc7a6119333dc87882344903b7 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Sat, 23 Dec 2023 12:13:02 -0800 Subject: [PATCH 36/41] Change to version 1.3.0 --- src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec b/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec index 805c3840..141101b3 100644 --- a/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec +++ b/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec @@ -2,7 +2,7 @@ Microsoft.NETCore.CoreDisTools - 1.2.0 + 1.3.0 Microsoft.NETCore Instruction-wise Disassembler Microsoft Microsoft From a5462970feabeebcb4950a68bfcdf0c1494bf357 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Fri, 29 Dec 2023 10:47:13 -0800 Subject: [PATCH 37/41] Update version --- src/coredistools/coredistools.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coredistools/coredistools.h b/src/coredistools/coredistools.h index 5016e836..2f631781 100644 --- a/src/coredistools/coredistools.h +++ b/src/coredistools/coredistools.h @@ -1,9 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -//===--------- coredistools.h - Dissassembly tools for CoreClr ------------===// +//===--------- coredistools.h - Disassembly tools for CoreClr ------------===// // -// Core Disassembly Tools API Version 1.2.0 +// Core Disassembly Tools API Version 1.3.0 // Disassembly tools required by CoreCLR for utilities like // GCStress, SuperPMI, and R2RDump. //===----------------------------------------------------------------------===// From d9b0b348d629dc173e012bd202c818615e33400e Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 2 Jan 2024 17:20:48 -0800 Subject: [PATCH 38/41] Bump version to 1.4.0 --- src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec | 2 +- src/coredistools/coredistools.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec b/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec index 141101b3..24cd6f3f 100644 --- a/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec +++ b/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec @@ -2,7 +2,7 @@ Microsoft.NETCore.CoreDisTools - 1.3.0 + 1.4.0 Microsoft.NETCore Instruction-wise Disassembler Microsoft Microsoft diff --git a/src/coredistools/coredistools.h b/src/coredistools/coredistools.h index 2f631781..20baa2d5 100644 --- a/src/coredistools/coredistools.h +++ b/src/coredistools/coredistools.h @@ -3,7 +3,7 @@ //===--------- coredistools.h - Disassembly tools for CoreClr ------------===// // -// Core Disassembly Tools API Version 1.3.0 +// Core Disassembly Tools API Version 1.4.0 // Disassembly tools required by CoreCLR for utilities like // GCStress, SuperPMI, and R2RDump. //===----------------------------------------------------------------------===// From 3d37674d58c3b64c50b7ae4b7f40973c63f0ebbd Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 2 Jan 2024 17:27:07 -0800 Subject: [PATCH 39/41] Restore x86 special prefix handling Looks like at least `lock` is still not treated as part of the instruction that follows it. --- src/coredistools/coredistools.cpp | 80 +++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/src/coredistools/coredistools.cpp b/src/coredistools/coredistools.cpp index 08867f65..3b7bdbe5 100644 --- a/src/coredistools/coredistools.cpp +++ b/src/coredistools/coredistools.cpp @@ -200,6 +200,17 @@ struct CorDisasm { unique_ptr Ctx; unique_ptr Disassembler; unique_ptr IP; + + // LLVM's MCInst does not expose Opcode enumerations by design. + // The following enumeration is a hack to use X86 opcode numbers, + // until bug 7709 is fixed. + struct OpcodeMap { + const char *Name; + uint8_t MachineOpcode; + }; + + static const int X86NumPrefixes = 19; + static const OpcodeMap X86Prefix[X86NumPrefixes]; }; struct CorAsmDiff : public CorDisasm { @@ -221,6 +232,30 @@ struct CorAsmDiff : public CorDisasm { OffsetMunger Munger; }; +// clang-format off +CorDisasm::OpcodeMap const CorDisasm::X86Prefix[CorDisasm::X86NumPrefixes] = { + { "LOCK", 0xF0 }, + { "REPNE/XACQUIRE", 0xF2 }, // Both the (TSX/normal) instrs + { "REP/XRELEASE", 0xF3 }, // have the same byte encoding + { "OP_OVR", 0x66 }, + { "CS_OVR", 0x2E }, + { "DS_OVR", 0x3E }, + { "ES_OVR", 0x26 }, + { "FS_OVR", 0x64 }, + { "GS_OVR", 0x65 }, + { "SS_OVR", 0x36 }, + { "ADDR_OVR", 0x67 }, + { "REX64W", 0x48 }, + { "REX64WB", 0x49 }, + { "REX64WX", 0x4A }, + { "REX64WXB", 0x4B }, + { "REX64WR", 0x4C }, + { "REX64WRB", 0x4D }, + { "REX64WRX", 0x4E }, + { "REX64WRXB", 0x4F } +}; +// clang-format on + bool CorDisasm::setTarget() { // Figure out the target triple. @@ -395,17 +430,46 @@ bool CorDisasm::decodeInstruction(BlockIterator &BIter, bool MayFail) const { uint64_t CorDisasm::disasmInstruction(BlockIterator &BIter, bool DumpAsm) const { - if (!decodeInstruction(BIter)) { - return 0; - } + uint64_t TotalSize = 0; + bool ContinueDisasm; - uint64_t Size = BIter.InstrSize; + // On X86, LLVM disassembler does not handle instruction prefixes + // correctly -- please see LLVM bug 7709. + // The disassembler reports instruction prefixes separate from the + // actual instruction. In order to work-around this problem, we + // continue decoding past the prefix bytes. - if (DumpAsm) { - dumpInstruction(BIter); - } + do { + + if (!decodeInstruction(BIter)) { + return 0; + } + + uint64_t Size = BIter.InstrSize; + TotalSize += Size; + + if (DumpAsm) { + dumpInstruction(BIter); + } + + ContinueDisasm = false; + if ((TheTargetArch == Target_X86) || (TheTargetArch == Target_X64)) { + + // Check if the decoded instruction is a prefix byte, and if so, + // continue decoding. + if (Size == 1) { + for (uint8_t Pfx = 0; Pfx < X86NumPrefixes; Pfx++) { + if (BIter.Ptr[0] == X86Prefix[Pfx].MachineOpcode) { + ContinueDisasm = true; + BIter.advance(); + break; + } + } + } + } + } while (ContinueDisasm); - return Size; + return TotalSize; } void CorDisasm::dumpInstruction(const BlockIterator &BIter) const { From e43879c282be3711c5fbcad12bcd21b4f72fa324 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 2 Jan 2024 17:28:02 -0800 Subject: [PATCH 40/41] Remove clang warning disable from header file --- src/coredistools/coredistools.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/coredistools/coredistools.h b/src/coredistools/coredistools.h index 20baa2d5..f26693f9 100644 --- a/src/coredistools/coredistools.h +++ b/src/coredistools/coredistools.h @@ -26,10 +26,6 @@ #define DllIface EXTERN_C __declspec(dllimport) #endif // defined(DllInterfaceExporter) #else - -// Disable "warning: default label in switch which covers all enumeration values [-Wcovered-switch-default]" -#pragma clang diagnostic ignored "-Wcovered-switch-default" - #if !defined(__cdecl) #if defined(__i386__) #define __cdecl __attribute__((cdecl)) From b8c1031308c0a17f4573b68032a1fb082988f8a4 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 2 Jan 2024 17:46:40 -0800 Subject: [PATCH 41/41] Fix clang build warning --- src/coredistools/coredistools.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/coredistools/coredistools.cpp b/src/coredistools/coredistools.cpp index 3b7bdbe5..0ff8232f 100644 --- a/src/coredistools/coredistools.cpp +++ b/src/coredistools/coredistools.cpp @@ -256,6 +256,11 @@ CorDisasm::OpcodeMap const CorDisasm::X86Prefix[CorDisasm::X86NumPrefixes] = { }; // clang-format on +#if !defined(_MSC_VER) +// Disable "warning: default label in switch which covers all enumeration values [-Wcovered-switch-default]" +#pragma clang diagnostic ignored "-Wcovered-switch-default" +#endif + bool CorDisasm::setTarget() { // Figure out the target triple.