tweaks to enable NativeAOT-LLVM on Linux with WASI-SDK 22 (#2592)

* tweaks to enable NativeAOT-LLVM on Linux with WASI-SDK 22

Some of these changes are borrowed from
#2569.

Note that I had to manually copy pthread.h from the
wasi-sdk-22/share/wasi-sysroot/include/wasm32-wasi-threads directory to the
wasi-sdk-22/share/wasi-sysroot/include/wasm32-wasi directory as a workaround
until WebAssembly/wasi-libc#501 is addressed.

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

* Restrict WASI SDK version check to WASI target

Co-authored-by: yowl <scott.waye@hubse.com>

---------

Signed-off-by: Joel Dice <joel.dice@fermyon.com>
Co-authored-by: yowl <scott.waye@hubse.com>

docs/workflow/building/coreclr/nativeaot.md

@@ -24,7 +24,7 @@ The Native AOT toolchain can be currently built for Linux (x64/arm64), macOS (x6
 There are two kinds of binary artifacts produced by the build and needed for development: the runtime libraries and the cross-targeting compilers, ILC and RyuJit. They are built differently and separately.
 
 For the runtime libraries:
-- Clone the [emsdk](https://github.com/emscripten-core/emsdk) repository and use the `emsdk.bat` script it comes with to [install](https://emscripten.org/docs/getting_started/downloads.html) (and optionally "activate", i. e. set the relevant environment variables permanently) the Emscripten SDK, which will be used by the native build as a sort of "virtualized" build environment. It is recommended to use the same Emscripten version that [the CI](https://github.com/dotnet/runtimelab/blob/feature/NativeAOT-LLVM/eng/pipelines/runtimelab/install-emscripten.ps1#L16-L20) uses.
+- To build for web browsers, clone the [emsdk](https://github.com/emscripten-core/emsdk) repository and use the `emsdk.bat` script it comes with to [install](https://emscripten.org/docs/getting_started/downloads.html) (and optionally "activate", i. e. set the relevant environment variables permanently) the Emscripten SDK, which will be used by the native build as a sort of "virtualized" build environment. It is recommended to use the same Emscripten version that [the CI](https://github.com/dotnet/runtimelab/blob/feature/NativeAOT-LLVM/eng/pipelines/runtimelab/install-emscripten.ps1#L16-L20) uses.
   ```
   git clone https://github.com/emscripten-core/emsdk
   cd emsdk
@@ -34,7 +34,7 @@ For the runtime libraries:
   ./emsdk install 3.1.47
   ./emsdk activate 3.1.47
   ```
-- To build for WASI, download and install the Wasi SDK from https://github.com/WebAssembly/wasi-sdk/releases (only Windows is supported currently) and set the `WASI_SDK_PATH` environment variable to the location where it is installed, e.g. `set WASI_SDK_PATH=c:\github\wasi-sdk`.
+- To build for WASI, download and install WASI-SDK 22 from https://github.com/WebAssembly/wasi-sdk/releases (only Windows and Linux are supported currently) and set the `WASI_SDK_PATH` environment variable to the location where it is installed, e.g. `set WASI_SDK_PATH=c:\github\wasi-sdk`. Note that WASI-SDK 22 only includes a copy of `pthread.h` for the `wasm32-wasi-threads` target, which we must copy to the include directory for the `wasm32-wasi` target, e.g. `cp $WASI_SDK\share\wasi-sysroot\include\wasm32-wasi-threads\pthread.h $WASI_SDK\share\wasi-sysroot\include\wasm32-wasi\`. This is a temporary workaround until https://github.com/WebAssembly/wasi-libc/issues/501 has been addressed and released.
 - Run `build clr.aot+libs -c [Debug|Release] -a wasm -os [browser|wasi]`. This will create the architecture-dependent libraries needed for linking and runtime execution, as well as the managed binaries to be used as input to ILC.
 
 For the compilers:

eng/native/gen-buildsys.sh

@@ -93,11 +93,17 @@ if [[ "$scan_build" == "ON" && -n "$SCAN_BUILD_COMMAND" ]]; then
     cmake_command="$SCAN_BUILD_COMMAND $cmake_command"
 fi
 
+cmake_extra_defines_wasm=()
 if [[ "$host_arch" == "wasm" ]]; then
     if [[ "$target_os" == "browser" ]]; then
         cmake_command="emcmake $cmake_command"
     elif [[ "$target_os" == "wasi" ]]; then
-        true
+        if [[ -z $WASI_SDK_PATH ]]; then
+            echo "Error: Should set WASI_SDK_PATH environment variable pointing to WASI SDK root."
+            exit 1
+        fi
+
+        cmake_extra_defines_wasm=("-DCLR_CMAKE_TARGET_OS=wasi" "-DCLR_CMAKE_TARGET_ARCH=wasm" "-DWASI_SDK_PREFIX=$WASI_SDK_PATH" "-DCMAKE_TOOLCHAIN_FILE=$WASI_SDK_PATH/share/cmake/wasi-sdk.cmake" "-DCMAKE_CROSSCOMPILING_EMULATOR=node --experimental-wasm-bigint --experimental-wasi-unstable-preview1")
     else
         echo "target_os was not specified"
         exit 1
@@ -110,6 +116,7 @@ $cmake_command \
   "-DCMAKE_BUILD_TYPE=$buildtype" \
   "-DCMAKE_INSTALL_PREFIX=$__CMakeBinDir" \
   $cmake_extra_defines \
+  "${cmake_extra_defines_wasm[@]}" \
   $__UnprocessedCMakeArgs \
   -S "$1" \
   -B "$2"

eng/pipelines/runtimelab/install-wasi-sdk.ps1

@@ -1,6 +1,13 @@
-Invoke-WebRequest -Uri https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0.m-mingw.tar.gz -OutFile wasi-sdk-20.0.m-mingw.tar.gz
+Invoke-WebRequest -Uri https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-22/wasi-sdk-22.0.m-mingw64.tar.gz -OutFile wasi-sdk-22.0.m-mingw64.tar.gz
 
-tar -xzf wasi-sdk-20.0.m-mingw.tar.gz
+tar -xzf wasi-sdk-22.0.m-mingw64.tar.gz
 
-mv wasi-sdk-20.0+m wasi-sdk
+mv wasi-sdk-22.0+m wasi-sdk
 
+# Temporary WASI-SDK 22 workaround: Until
+# https://github.com/WebAssembly/wasi-libc/issues/501 is addressed, we copy
+# pthread.h from the wasm32-wasi-threads include directory to the wasm32-wasi
+# include directory.  See https://github.com/dotnet/runtimelab/issues/2598 for
+# the issue to remove this workaround once WASI-SDK 23 is released.
+
+cp wasi-sdk/share/wasi-sysroot/include/wasm32-wasi-threads/pthread.h wasi-sdk/share/wasi-sysroot/include/wasm32-wasi/

src/coreclr/build-runtime.sh

@@ -177,11 +177,11 @@ if [[ "$__CMakeTarget" == *"wasmjit"* ]]; then
 
   if [[ "$__BuildType" == "Release" ]]; then
     if [[ -n $LLVM_CMAKE_CONFIG_RELEASE ]]; then
-      LLVM_CMAKE_CONFIG="$LLVM_CMAKE_CONFIG_RELEASE"
+      export LLVM_CMAKE_CONFIG="$LLVM_CMAKE_CONFIG_RELEASE"
     fi
   else
     if [[ -n $LLVM_CMAKE_CONFIG_DEBUG ]]; then
-      LLVM_CMAKE_CONFIG="$LLVM_CMAKE_CONFIG_DEBUG"
+      export LLVM_CMAKE_CONFIG="$LLVM_CMAKE_CONFIG_DEBUG"
     fi
   fi
 

src/coreclr/jit/CMakeLists.txt

@@ -706,16 +706,11 @@ else()
   create_standalone_jit(TARGET clrjit_win_x86_${ARCH_HOST_NAME} OS win ARCH x86 DESTINATIONS .)
 endif (CLR_CMAKE_TARGET_ARCH_RISCV64)
 
-# Note that we currently do not support building the LLVM Jit on Unix.
-# Note as well that we need this "CLR_CMAKE_BUILD_LLVM_JIT", defined by the build scripts,
+# Note that we need this "CLR_CMAKE_BUILD_LLVM_JIT", defined by the build scripts,
 # because there is no way in CMake to ask "what --target's' am I being asked to configure?".
 if (CLR_CMAKE_BUILD_LLVM_JIT)
     # The LLVM clrjit needs to be the last clrjit to use create_standalone_jit as it modifies some cmake variables.
     # LLVM clrjit has an extra export - registerLlvmCallbacks.
-    set(CLRJIT_EXPORTS ${CMAKE_CURRENT_LIST_DIR}/ClrJit.Llvm.exports)
-    set(JIT_EXPORTS_FILE ${CMAKE_CURRENT_BINARY_DIR}/ClrJit.Llvm.exports.def)
-    preprocess_file (${CLRJIT_EXPORTS} ${JIT_EXPORTS_FILE})
-    set(JIT_DEF_FILE ${JIT_EXPORTS_FILE})
 
     # Exclude cpp files that are not required when not processing beyond rationalized LIR.
     # Use REGEX as this list contains the absolute paths.

src/coreclr/jit/ClrJit.PAL.exports

@@ -3,3 +3,4 @@ jitStartup
 getLikelyClasses
 getLikelyMethods
 jitBuildString
+registerLlvmCallbacks

src/coreclr/jit/ClrJit.exports

@@ -7,3 +7,4 @@ EXPORTS
     getLikelyClasses
     getLikelyMethods
     jitBuildString
+    registerLlvmCallbacks

src/coreclr/jit/ee_il_dll.cpp

@@ -1421,3 +1421,10 @@ unsigned Compiler::eeTryGetClassSize(CORINFO_CLASS_HANDLE clsHnd)
 }
 
 #endif // !DEBUG
+
+#ifndef TARGET_WASM
+extern "C" DLLEXPORT void registerLlvmCallbacks(void** jitImports, void** jitExports)
+{
+    // No-op stub; see llvm.cpp for the real implementation for `TARGET_WASM`
+}
+#endif

src/coreclr/jit/llvm.cpp

@@ -770,7 +770,7 @@ CORINFO_GENERIC_HANDLE Llvm::getSymbolHandleForClassToken(mdToken token)
 template <EEApiId Func, typename TReturn, typename... TArgs>
 TReturn CallEEApi(TArgs... args)
 {
-    return static_cast<TReturn (*)(TArgs...)>(g_callbacks[static_cast<int>(Func)])(args...);
+    return reinterpret_cast<TReturn (*)(TArgs...)>(g_callbacks[static_cast<int>(Func)])(args...);
 }
 
 const char* Llvm::GetMangledMethodName(CORINFO_METHOD_HANDLE methodHandle)
@@ -866,8 +866,8 @@ extern "C" DLLEXPORT void registerLlvmCallbacks(void** jitImports, void** jitExp
     assert(jitExports != nullptr);
 
     memcpy(g_callbacks, jitImports, static_cast<int>(EEApiId::Count) * sizeof(void*));
-    jitExports[static_cast<int>(JitApiId::StartSingleThreadedCompilation)] = &Llvm::StartSingleThreadedCompilation;
-    jitExports[static_cast<int>(JitApiId::FinishSingleThreadedCompilation)] = &Llvm::FinishSingleThreadedCompilation;
+    jitExports[static_cast<int>(JitApiId::StartSingleThreadedCompilation)] = (void*)&Llvm::StartSingleThreadedCompilation;
+    jitExports[static_cast<int>(JitApiId::FinishSingleThreadedCompilation)] = (void*)&Llvm::FinishSingleThreadedCompilation;
     jitExports[static_cast<int>(JitApiId::Count)] = (void*)0x1234;
 }
 

src/coreclr/jit/llvmcodegen.cpp

@@ -1118,6 +1118,7 @@ void Llvm::visitNode(GenTree* node)
             break;
         case GT_JMP:
             NYI("LLVM/GT_JMP"); // Requires support for explicit tailcalls.
+            break;
         default:
             unreached();
     }

src/coreclr/jit/llvmlower.cpp

@@ -409,6 +409,9 @@ void Llvm::lowerFieldOfDependentlyPromotedStruct(GenTree* node)
                         lclVar->gtFlags |= GTF_VAR_USEASG;
                     }
                     break;
+
+                default:
+                    break;
             }
 
             lclVar->SetLclNum(varDsc->lvParentLcl);

src/coreclr/jit/llvmlssa.cpp

@@ -1482,7 +1482,7 @@ class ShadowStackAllocator
         {
             if (block != m_currentBlock)
             {
-                m_actions.Push({AllocationActionKind::Block, m_currentBlockIndex++});
+                m_actions.Push({AllocationActionKind::Block, {m_currentBlockIndex++}});
                 m_currentBlock = block;
             }
         }
@@ -1636,6 +1636,10 @@ class ShadowStackAllocator
         template <typename... TArgs>
         void PrintFormatted(char** pBuffer, const char* format, TArgs... args)
         {
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat-security"
+#endif // __clang__
             if (pBuffer == nullptr)
             {
                 printf(format, args...);
@@ -1644,6 +1648,9 @@ class ShadowStackAllocator
             {
                 *pBuffer += sprintf(*pBuffer, format, args...);
             }
+#ifdef __clang__
+#pragma clang diagnostic pop            
+#endif // __clang__
         }
 
         void PrintAction(const AllocationAction& action, char** pBuffer = nullptr)

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets

@@ -388,9 +388,11 @@ The .NET Foundation licenses this file to you under the MIT license.
 
   <Target Name="CheckWasmSdks">
     <Error Text="Emscripten not found, not compiling to WebAssembly. To enable WebAssembly compilation, install Emscripten and ensure the EMSDK environment variable points to the directory containing upstream/emscripten/emcc.bat"
-           Condition="'$(EMSDK)' == ''" />
+           Condition="'$(EMSDK)' == '' and '$(_targetOS)' == 'browser'" />
     <Error Text="Wasi SDK not found, not compiling to WebAssembly. To enable WebAssembly compilation, install Wasi SDK and ensure the WASI_SDK_PATH environment variable points to the directory containing share/wasi-sysroot"
            Condition="'$(WASI_SDK_PATH)' == '' and '$(_targetOS)' == 'wasi'" />
+    <Warning Text="The WASI SDK version is too low. Please use WASI SDK 22 or newer with a 64 bit Clang."
+             Condition="!Exists('$(WASI_SDK_PATH)/VERSION') and '$(_targetOS)' == 'wasi'" />
   </Target>
 
   <Target Name="CompileWasmObjects"
@@ -420,8 +422,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <CompileWasmArgs>$(CompileWasmArgs) -fvisibility=default -mllvm -combiner-global-alias-analysis=false -mllvm -disable-lsr --sysroot=&quot;$(WASI_SDK_PATH)/share/wasi-sysroot&quot; -target $(IlcLlvmTarget)</CompileWasmArgs>
 
       <ExeExt Condition="'$(OS)' == 'Windows_NT'">.exe</ExeExt>
-      <!-- using Emscripten's clang++ because of a crash in wasi-sdk's clang++ (https://github.com/WebAssembly/wasi-sdk/issues/326) -->
-      <WasmCompilerPath>&quot;$(EMSDK)/upstream/bin/clang++$(ExeExt)&quot;</WasmCompilerPath>
+      <WasmCompilerPath>&quot;$(WASI_SDK_PATH)/bin/clang++$(ExeExt)&quot;</WasmCompilerPath>
       <WasmLinkerPath>&quot;$(WASI_SDK_PATH)/bin/clang&quot;</WasmLinkerPath>
     </PropertyGroup>
 

src/native/libs/CMakeLists.txt

@@ -32,6 +32,9 @@ if (CLR_CMAKE_TARGET_UNIX OR CLR_CMAKE_TARGET_BROWSER OR CLR_CMAKE_TARGET_WASI)
     set(CMAKE_INSTALL_PREFIX $ENV{__CMakeBinDir})
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99 -Wno-declaration-after-statement")
 
+    # TODO-LLVM: remove once upstream moves to WASI SDK 22 as well (or otherwise fixes these warnings).
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-switch-default")
+
     add_compile_options(-I${CMAKE_CURRENT_SOURCE_DIR}/Common)
     add_compile_options(-I${CMAKE_CURRENT_BINARY_DIR}/Common)