Static linking of native libs (#44505)

* from prototype

* fix OSX

* fix for Android

* treat "libSystem.Globalization.Native" as QCall in mono too (for now).

* fix for wasm

* fix pedantic errors on GCC

* delete gPalGlobalizationNative

* pass overrider from the host

* default override

* default PInvoke override runs after optional host-provided overrider.

* Some PR feedback   (mostly related to code, not the CMake stuff).

* more coding PR feedback

* Deleted "libraries-native" folder.

* unifying tryrun.cmake into 1 common file

* factor out adding lib-specific dependencies into one place (per native library)

* cleanup: entirely remove tryrun.cmake propagation in eng, gen-buildsys, build-commons, and build scripts

* remove "clrcompression.dll" and "libSystem.IO.Compression.Native.dylib" from single-file host packaging

* mono: refactor Globalization lookup into `default_resolve_dllimport`.

* set FEATURE_DISTRO_AGNOSTIC_SSL according to __PortableBuild

* CORECLR_CALLING_CONVENTION for the host callback types

* refactor common parts of entrypoints.c into entrypoints.h

* rename OverrideEntry -->  DllImportEntry

* extra libs as CMake functions

* use macros instead of functions in extra_libs.cmake

* mono formatting style in src/mono/mono/metadata/native-library.c

Co-authored-by: Ryan Lucia <ryan@luciaonline.net>

* mono: Remaining stylistic nits.

Co-authored-by: Ryan Lucia <ryan@luciaonline.net>

* do not set DFEATURE_DISTRO_AGNOSTIC_SSL on iOS, Android, tvOS

Co-authored-by: Ryan Lucia <ryan@luciaonline.net>

eng/native/build-commons.sh

@@ -71,10 +71,9 @@ build_native()
  targetOS="$1"
  platformArch="$2"
  cmakeDir="$3"
- tryrunDir="$4"
- intermediatesDir="$5"
- cmakeArgs="$6"
- message="$7"
+ intermediatesDir="$4"
+ cmakeArgs="$5"
+ message="$6"
 
  # All set to commence the build
  echo "Commencing build of \"$message\" for $__TargetOS.$__BuildArch.$__BuildType in $intermediatesDir"
@@ -146,7 +145,7 @@ EOF
  scan_build=scan-build
  fi
 
- nextCommand="\"$__RepoRootDir/eng/native/gen-buildsys.sh\" \"$cmakeDir\" \"$tryrunDir\" \"$intermediatesDir\" $platformArch $__Compiler \"$__CompilerMajorVersion\" \"$__CompilerMinorVersion\" $__BuildType \"$generator\" $scan_build $cmakeArgs"
+ nextCommand="\"$__RepoRootDir/eng/native/gen-buildsys.sh\" \"$cmakeDir\" \"$intermediatesDir\" $platformArch $__Compiler \"$__CompilerMajorVersion\" \"$__CompilerMinorVersion\" $__BuildType \"$generator\" $scan_build $cmakeArgs"
  echo "Invoking $nextCommand"
  eval $nextCommand
 
@@ -455,6 +454,22 @@ if [[ "$__PortableBuild" == 0 ]]; then
  __CommonMSBuildArgs="$__CommonMSBuildArgs /p:PortableBuild=false"
 fi
 
+if [[ "$__BuildArch" == wasm ]]; then
+ # nothing to do here
+ true
+elif [[ "$__TargetOS" == iOS ]]; then
+ # nothing to do here
+ true
+elif [[ "$__TargetOS" == tvOS ]]; then
+ # nothing to do here
+ true
+elif [[ "$__TargetOS" == Android ]]; then
+ # nothing to do here
+ true
+else
+ __CMakeArgs="-DFEATURE_DISTRO_AGNOSTIC_SSL=$__PortableBuild $__CMakeArgs"
+fi
+
 # Configure environment if we are doing a cross compile.
 if [[ "$__CrossBuild" == 1 ]]; then
  CROSSCOMPILE=1

eng/native/configurecompiler.cmake

@@ -451,7 +451,6 @@ endif(CLR_CMAKE_HOST_UNIX)
 if (MSVC)
  # Compile options for targeting windows
 
- add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/TP>) # compile all files as C++
  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/nologo>) # Suppress Startup Banner
  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/W3>) # set warning level to 3
  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/WX>) # treat warnings as errors

eng/native/gen-buildsys.sh

@@ -7,10 +7,9 @@ scriptroot="$( cd -P "$( dirname "$0" )" && pwd )"
 
 if [[ "$#" -lt 4 ]]; then
  echo "Usage..."
- echo "gen-buildsys.sh <path to top level CMakeLists.txt> <path to tryrun.cmake directory> <path to intermediate directory> <Architecture> <compiler> <compiler major version> <compiler minor version> [build flavor] [ninja] [scan-build] [cmakeargs]"
+ echo "gen-buildsys.sh <path to top level CMakeLists.txt> <path to intermediate directory> <Architecture> <compiler> <compiler major version> <compiler minor version> [build flavor] [ninja] [scan-build] [cmakeargs]"
  echo "Specify the path to the top level CMake file."
  echo "Specify the path that the build system files are generated in."
- echo "Specify the path to the directory with tryrun.cmake file."
  echo "Specify the target architecture."
  echo "Specify the name of compiler (clang or gcc)."
  echo "Specify the major version of compiler."
@@ -22,11 +21,10 @@ if [[ "$#" -lt 4 ]]; then
  exit 1
 fi
 
-tryrun_dir="$2"
-build_arch="$4"
-compiler="$5"
-majorVersion="$6"
-minorVersion="$7"
+build_arch="$3"
+compiler="$4"
+majorVersion="$5"
+minorVersion="$6"
 
 source "$scriptroot/init-compiler.sh" "$build_arch" "$compiler" "$majorVersion" "$minorVersion"
 
@@ -42,7 +40,7 @@ scan_build=OFF
 generator="Unix Makefiles"
 __UnprocessedCMakeArgs=""
 
-for i in "${@:8}"; do
+for i in "${@:7}"; do
  upperI="$(echo "$i" | awk '{print toupper($0)}')"
  case "$upperI" in
  # Possible build types are DEBUG, CHECKED, RELEASE, RELWITHDEBINFO.
@@ -73,9 +71,7 @@ if [[ "$CROSSCOMPILE" == "1" ]]; then
  TARGET_BUILD_ARCH="$build_arch"
  export TARGET_BUILD_ARCH
 
- if [[ -n "$tryrun_dir" ]]; then
- cmake_extra_defines="$cmake_extra_defines -C $tryrun_dir/tryrun.cmake"
- fi
+ cmake_extra_defines="$cmake_extra_defines -C $scriptroot/tryrun.cmake"
 
  if [[ "$platform" == "Darwin" ]]; then
  cmake_extra_defines="$cmake_extra_defines -DCMAKE_SYSTEM_NAME=Darwin"
@@ -99,7 +95,7 @@ if [[ "$build_arch" == "wasm" ]]; then
 fi
 
 # We have to be able to build with CMake 3.6.2, so we can't use the -S or -B options
-pushd "$3"
+pushd "$2"
 
 # Include CMAKE_USER_MAKE_RULES_OVERRIDE as uninitialized since it will hold its value in the CMake cache otherwise can cause issues when branch switching
 $cmake_command \

src/coreclr/tryrun.cmake → eng/native/tryrun.cmake

@@ -31,6 +31,7 @@ if(DARWIN)
  set_cache_value(HAVE_BROKEN_FIFO_SELECT_EXITCODE 1)
  set_cache_value(HAVE_CLOCK_MONOTONIC_COARSE_EXITCODE 1)
  set_cache_value(HAVE_CLOCK_MONOTONIC_EXITCODE 0)
+ set_cache_value(HAVE_CLOCK_REALTIME_EXITCODE 0)
  set_cache_value(HAVE_CLOCK_THREAD_CPUTIME_EXITCODE 0)
  set_cache_value(HAVE_CLOCK_GETTIME_NSEC_NP_EXITCODE 0)
  set_cache_value(HAVE_COMPATIBLE_ACOS_EXITCODE 0)
@@ -63,6 +64,7 @@ if(DARWIN)
  set_cache_value(SSCANF_CANNOT_HANDLE_MISSING_EXPONENT_EXITCODE 1)
  set_cache_value(SSCANF_SUPPORT_ll_EXITCODE 0)
  set_cache_value(UNGETC_NOT_RETURN_EOF_EXITCODE 1)
+ set_cache_value(HAVE_SHM_OPEN_THAT_WORKS_WELL_ENOUGH_WITH_MMAP_EXITCODE 0)
  else()
  message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm64 or x64 is supported for OSX cross build!")
  endif()
@@ -72,6 +74,7 @@ elseif(TARGET_ARCH_NAME MATCHES "^(armel|arm|arm64|x86)$" OR FREEBSD OR ILLUMOS)
  set_cache_value(HAS_POSIX_SEMAPHORES_EXITCODE 0)
  set_cache_value(HAVE_CLOCK_MONOTONIC_COARSE_EXITCODE 0)
  set_cache_value(HAVE_CLOCK_MONOTONIC_EXITCODE 0)
+ set_cache_value(HAVE_CLOCK_REALTIME_EXITCODE 0)
  set_cache_value(HAVE_CLOCK_THREAD_CPUTIME_EXITCODE 0)
  set_cache_value(HAVE_COMPATIBLE_ACOS_EXITCODE 0)
  set_cache_value(HAVE_COMPATIBLE_ASIN_EXITCODE 0)
@@ -100,16 +103,21 @@ elseif(TARGET_ARCH_NAME MATCHES "^(armel|arm|arm64|x86)$" OR FREEBSD OR ILLUMOS)
 
 
  if(ALPINE_LINUX)
+ set_cache_value(HAVE_SHM_OPEN_THAT_WORKS_WELL_ENOUGH_WITH_MMAP_EXITCODE 1)
  set_cache_value(SSCANF_CANNOT_HANDLE_MISSING_EXPONENT_EXITCODE 0)
  set_cache_value(SSCANF_SUPPORT_ll_EXITCODE 1)
  set_cache_value(UNGETC_NOT_RETURN_EOF_EXITCODE 1)
  else()
+ set_cache_value(HAVE_SHM_OPEN_THAT_WORKS_WELL_ENOUGH_WITH_MMAP_EXITCODE 0)
  set_cache_value(SSCANF_CANNOT_HANDLE_MISSING_EXPONENT_EXITCODE 1)
  set_cache_value(SSCANF_SUPPORT_ll_EXITCODE 0)
  set_cache_value(UNGETC_NOT_RETURN_EOF_EXITCODE 0)
  endif()
 
  if (FREEBSD)
+ set_cache_value(HAVE_SHM_OPEN_THAT_WORKS_WELL_ENOUGH_WITH_MMAP 1)
+ set_cache_value(HAVE_CLOCK_MONOTONIC 1)
+ set_cache_value(HAVE_CLOCK_REALTIME 1)
  set_cache_value(HAVE_BROKEN_FIFO_KEVENT_EXITCODE 1)
  set_cache_value(HAVE_PROCFS_MAPS 0)
  set_cache_value(HAVE_PROCFS_STAT 0)

src/coreclr/CMakeLists.txt

@@ -52,7 +52,13 @@ include(pgosupport.cmake)
 # Include libraries native shims
 #-------------------------------
 if(NOT CLR_CROSS_COMPONENTS_BUILD AND CLR_CMAKE_BUILD_SUBSET_RUNTIME)
- add_subdirectory(src/libraries-native)
+ set(STATIC_LIBS_ONLY 1)
+
+ if(CLR_CMAKE_TARGET_WIN32)
+ add_subdirectory(${CLR_REPO_ROOT_DIR}/src/libraries/Native/Windows Native.Windows)
+ else()
+ add_subdirectory(${CLR_REPO_ROOT_DIR}/src/libraries/Native/Unix Native.Unix)
+ endif()
 endif(NOT CLR_CROSS_COMPONENTS_BUILD AND CLR_CMAKE_BUILD_SUBSET_RUNTIME)
 
 #-----------------------------------------

src/coreclr/build-runtime.sh

@@ -99,7 +99,7 @@ build_cross_architecture_components()
  export __CMakeBinDir CROSSCOMPILE
 
  __CMakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CROSS_COMPONENTS_BUILD=1 $__CMakeArgs"
- build_native "$__TargetOS" "$__CrossArch" "$__ProjectRoot" "$__ProjectRoot" "$intermediatesForBuild" "$__CMakeArgs" "cross-architecture components"
+ build_native "$__TargetOS" "$__CrossArch" "$__ProjectRoot" "$intermediatesForBuild" "$__CMakeArgs" "cross-architecture components"
 
  CROSSCOMPILE=1
  export CROSSCOMPILE
@@ -257,7 +257,7 @@ fi
 if [[ "$__SkipNative" == 1 ]]; then
  echo "Skipping CoreCLR component build."
 else
- build_native "$__TargetOS" "$__BuildArch" "$__ProjectRoot" "$__ProjectRoot" "$__IntermediatesDir" "$__CMakeArgs" "CoreCLR component"
+ build_native "$__TargetOS" "$__BuildArch" "$__ProjectRoot" "$__IntermediatesDir" "$__CMakeArgs" "CoreCLR component"
 
  # Build cross-architecture components
  if [[ "$__SkipCrossArchNative" != 1 ]]; then

src/coreclr/src/CMakeLists.txt

@@ -7,6 +7,7 @@ include_directories("classlibnative/bcltype")
 include_directories("classlibnative/cryptography")
 include_directories("classlibnative/inc")
 include_directories("${GENERATED_INCLUDE_DIR}")
+include_directories("hosts/inc")
 
 if(CLR_CMAKE_TARGET_WIN32 AND FEATURE_EVENT_TRACE)
  include_directories("${GENERATED_INCLUDE_DIR}/etw")

src/coreclr/src/dlls/mscoree/unixinterface.cpp

@@ -18,6 +18,7 @@
 #include "../../vm/gdbjithelpers.h"
 #endif // FEATURE_GDBJIT
 #include "bundle.h"
+#include "pinvokeoverride.h"
 
 #define ASSERTE_ALL_BUILDS(expr) _ASSERTE_ALL_BUILDS(__FILE__, (expr))
 
@@ -119,7 +120,8 @@ static void ConvertConfigPropertiesToUnicode(
  int propertyCount,
  LPCWSTR** propertyKeysWRef,
  LPCWSTR** propertyValuesWRef,
- BundleProbe** bundleProbe,
+ BundleProbeFn** bundleProbe,
+ PInvokeOverrideFn** pinvokeOverride,
  bool* hostPolicyEmbedded)
 {
  LPCWSTR* propertyKeysW = new (nothrow) LPCWSTR[propertyCount];
@@ -137,7 +139,13 @@ static void ConvertConfigPropertiesToUnicode(
  {
  // If this application is a single-file bundle, the bundle-probe callback 
  // is passed in as the value of "BUNDLE_PROBE" property (encoded as a string).
- *bundleProbe = (BundleProbe*)_wcstoui64(propertyValuesW[propertyIndex], nullptr, 0);
+ *bundleProbe = (BundleProbeFn*)_wcstoui64(propertyValuesW[propertyIndex], nullptr, 0);
+ }
+ else if (strcmp(propertyKeys[propertyIndex], "PINVOKE_OVERRIDE") == 0)
+ {
+ // If host provides a PInvoke override (typically in a single-file bundle),
+ // the override callback is passed in as the value of "PINVOKE_OVERRIDE" property (encoded as a string).
+ *pinvokeOverride = (PInvokeOverrideFn*)_wcstoui64(propertyValuesW[propertyIndex], nullptr, 0);
  }
  else if (strcmp(propertyKeys[propertyIndex], "HOSTPOLICY_EMBEDDED") == 0)
  {
@@ -185,8 +193,9 @@ int coreclr_initialize(
 
  LPCWSTR* propertyKeysW;
  LPCWSTR* propertyValuesW;
- BundleProbe* bundleProbe = nullptr;
+ BundleProbeFn* bundleProbe = nullptr;
  bool hostPolicyEmbedded = false;
+ PInvokeOverrideFn* pinvokeOverride = nullptr;
 
  ConvertConfigPropertiesToUnicode(
  propertyKeys,
@@ -195,6 +204,7 @@ int coreclr_initialize(
  &propertyKeysW,
  &propertyValuesW,
  &bundleProbe,
+ &pinvokeOverride,
  &hostPolicyEmbedded);
 
 #ifdef TARGET_UNIX
@@ -211,6 +221,11 @@ int coreclr_initialize(
 
  g_hostpolicy_embedded = hostPolicyEmbedded;
 
+ if (pinvokeOverride != nullptr)
+ {
+ PInvokeOverride::SetPInvokeOverride(pinvokeOverride);
+ }
+
  ReleaseHolder<ICLRRuntimeHost4> host;
 
  hr = CorHost2::CreateObject(IID_ICLRRuntimeHost4, (void**)&host);

src/coreclr/src/hosts/inc/coreclrhost.h

@@ -14,6 +14,8 @@
 #define CORECLR_CALLING_CONVENTION
 #endif
 
+#include <stdint.h>
+
 // For each hosting API, we define a function prototype and a function pointer
 // The prototype is useful for implicit linking against the dynamic coreclr
 // library and the pointer for explicit dynamic loading (dlopen, LoadLibrary)
@@ -121,4 +123,11 @@ CORECLR_HOSTING_API(coreclr_execute_assembly,
 
 #undef CORECLR_HOSTING_API
 
+//
+// Callback types used by the hosts
+//
+typedef bool(CORECLR_CALLING_CONVENTION BundleProbeFn)(const char* path, int64_t* offset, int64_t* size);
+typedef const void* (CORECLR_CALLING_CONVENTION PInvokeOverrideFn)(const char* libraryName, const char* entrypointName);
+
+
 #endif // __CORECLR_HOST_H__

src/coreclr/src/inc/bundle.h

@@ -3,14 +3,15 @@
 
 /*****************************************************************************
  ** **
- ** bundle.h - Information about applications bundled as a single-file **
+ ** bundle.h - Information about applications bundled as a single-file  **
  ** **
  *****************************************************************************/
 
 #ifndef _BUNDLE_H_
 #define _BUNDLE_H_
 
 #include <sstring.h>
+#include "coreclrhost.h"
 
 class Bundle;
 
@@ -34,12 +35,10 @@ struct BundleFileLocation
  bool IsValid() const { LIMITED_METHOD_CONTRACT; return Offset != 0; }
 };
 
-typedef bool(__stdcall BundleProbe)(LPCSTR, INT64*, INT64*);
-
 class Bundle
 {
 public:
- Bundle(LPCSTR bundlePath, BundleProbe *probe);
+ Bundle(LPCSTR bundlePath, BundleProbeFn *probe);
  BundleFileLocation Probe(const SString& path, bool pathIsBundleRelative = false) const;
 
  const SString &Path() const { LIMITED_METHOD_CONTRACT; return m_path; }
@@ -52,11 +51,10 @@ class Bundle
 private:
 
  SString m_path; // The path to single-file executable
- BundleProbe *m_probe;
+ BundleProbeFn *m_probe;
 
  SString m_basePath; // The prefix to denote a path within the bundle
  COUNT_T m_basePathLength;
 };
 
 #endif // _BUNDLE_H_
-// EOF =======================================================================

src/coreclr/src/inc/cor.h

@@ -2388,4 +2388,3 @@ inline ULONG CorSigUncompressPointer( // return number of bytes of that compre
 #endif // __cplusplus
 
 #endif // _COR_H_
-// EOF =======================================================================

src/coreclr/src/inc/corpriv.h

@@ -383,5 +383,4 @@ struct CORCOMPILE_DEPENDENCY;
 typedef GUID CORCOMPILE_NGEN_SIGNATURE;
 
 #endif // _CORPRIV_H_
-// EOF =======================================================================
 

src/coreclr/src/inc/pinvokeoverride.h

@@ -0,0 +1,22 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+/*****************************************************************************
+ ** **
+ ** pinvokeoverride.h - PInvoke binding override **
+ ** **
+ *****************************************************************************/
+
+#ifndef _PINVOKEOVERRIDE_H_
+#define _PINVOKEOVERRIDE_H_
+
+#include "coreclrhost.h"
+
+class PInvokeOverride
+{
+public:
+ static void SetPInvokeOverride(PInvokeOverrideFn* overrideImpl);
+ static const void* GetMethodImpl(const char* libraryName, const char* entrypointName);
+};
+
+#endif // _PINVOKEOVERRIDE_H_

src/coreclr/src/vm/CMakeLists.txt

@@ -102,6 +102,7 @@ set(VM_SOURCES_DAC_AND_WKS_COMMON
  perfmap.cpp
  perfinfo.cpp
  pgo.cpp
+ pinvokeoverride.cpp
  precode.cpp
  prestub.cpp
  profilerdiagnosticprotocolhelper.cpp

src/coreclr/src/vm/bundle.cpp

@@ -30,7 +30,7 @@ const SString &BundleFileLocation::Path() const
  return Bundle::AppBundle->Path();
 }
 
-Bundle::Bundle(LPCSTR bundlePath, BundleProbe *probe)
+Bundle::Bundle(LPCSTR bundlePath, BundleProbeFn *probe)
 {
  STANDARD_VM_CONTRACT;
 

src/coreclr/src/vm/corelib.cpp

@@ -371,9 +371,6 @@ const USHORT c_nCoreLibFieldDescriptions = NumItems(c_rgCoreLibFieldDescriptions
 // ECalls
 //
 
-// ECalls defined by libraries-native shims
-EXTERN_C const LPVOID gPalGlobalizationNative[];
-
 // When compiling crossgen, we only need the target version of the ecall tables
 #if !defined(CROSSGEN_COMPILE) || defined(CROSSGEN_CORELIB)