Allow interop resolvers to return self handle (#78004)

When user-defined resolvers return self handle via `GetMainProgramHandle()`, we lookup the cached handle in mono and fail to find one because `internal_module` is not cached in `native_library_module_map`.

Fix is to test the resolver returned handle against self; before the map lookup (which happens under the lock).

Close #77985.

* Allow interop resolvers to return self handle

* Disable new test on windows and monointerpreter

* Add new test to monointerpreter ExcludeList

src/mono/mono/metadata/native-library.c

@@ -584,6 +584,40 @@ netcore_probe_for_module_nofail (MonoImage *image, const char *file_name, int fl
 	return result;
 }
 
+static MonoDl*
+netcore_lookup_self_native_handle (void)
+{
+	ERROR_DECL (load_error);
+	if (!internal_module)
+		internal_module = mono_dl_open_self (load_error);
+
+	if (!internal_module)
+		mono_trace (G_LOG_LEVEL_INFO, MONO_TRACE_DLLIMPORT, "DllImport error loading library '__Internal': '%s'.", mono_error_get_message_without_fields (load_error));
+
+	mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_DLLIMPORT, "Native library found via __Internal.");
+	mono_error_cleanup (load_error);
+
+	return internal_module;
+}
+
+static MonoDl* native_handle_lookup_wrapper (gpointer handle)
+{
+	MonoDl *result = NULL;
+
+	if (!internal_module)
+		netcore_lookup_self_native_handle ();
+
+	if (internal_module->handle == handle) {
+		result = internal_module;
+	} else {
+		native_library_lock ();
+		result = netcore_handle_lookup (handle);
+		native_library_unlock ();
+	}
+
+	return result;
+}
+
 static MonoDl *
 netcore_resolve_with_dll_import_resolver (MonoAssemblyLoadContext *alc, MonoAssembly *assembly, const char *scope, guint32 flags, MonoError *error)
 {
@@ -631,9 +665,7 @@ netcore_resolve_with_dll_import_resolver (MonoAssemblyLoadContext *alc, MonoAsse
 	mono_runtime_invoke_checked (resolve, NULL, args, error);
 	goto_if_nok (error, leave);
 
-	native_library_lock ();
-	result = netcore_handle_lookup (lib);
-	native_library_unlock ();
+	result = native_handle_lookup_wrapper (lib);
 
 leave:
 	HANDLE_FUNCTION_RETURN_VAL (result);
@@ -688,9 +720,7 @@ netcore_resolve_with_load (MonoAssemblyLoadContext *alc, const char *scope, Mono
 	mono_runtime_invoke_checked (resolve, NULL, args, error);
 	goto_if_nok (error, leave);
 
-	native_library_lock ();
-	result = netcore_handle_lookup (lib);
-	native_library_unlock ();
+	result = native_handle_lookup_wrapper (lib);
 
 leave:
 	HANDLE_FUNCTION_RETURN_VAL (result);
@@ -755,9 +785,7 @@ netcore_resolve_with_resolving_event (MonoAssemblyLoadContext *alc, MonoAssembly
 	mono_runtime_invoke_checked (resolve, NULL, args, error);
 	goto_if_nok (error, leave);
 
-	native_library_lock ();
-	result = netcore_handle_lookup (lib);
-	native_library_unlock ();
+	result = native_handle_lookup_wrapper (lib);
 
 leave:
 	HANDLE_FUNCTION_RETURN_VAL (result);
@@ -802,22 +830,6 @@ netcore_check_alc_cache (MonoAssemblyLoadContext *alc, const char *scope)
 	return result;
 }
 
-static MonoDl*
-netcore_lookup_self_native_handle (void)
-{
-	ERROR_DECL (load_error);
-	if (!internal_module)
-		internal_module = mono_dl_open_self (load_error);
-
-	if (!internal_module)
-		mono_trace (G_LOG_LEVEL_INFO, MONO_TRACE_DLLIMPORT, "DllImport error loading library '__Internal': '%s'.", mono_error_get_message_without_fields (load_error));
-
-	mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_DLLIMPORT, "Native library found via __Internal.");
-	mono_error_cleanup (load_error);
-
-	return internal_module;
-}
-
 static MonoDl *
 netcore_lookup_native_library (MonoAssemblyLoadContext *alc, MonoImage *image, const char *scope, guint32 flags)
 {

src/mono/mono/utils/mono-dl.c

@@ -176,8 +176,9 @@ fix_libc_name (const char *name)
  * mono_dl_open_self:
  * \param error pointer to MonoError
  *
- * Returns a handle to the main program, on android x86 it's not possible to
- * call dl_open(null), it returns a null handle, so this function returns RTLD_DEFAULT
+ * Returns a handle to the main program, on Android it's not possible to
+ * call dl_open(null) with RTLD_LAZY, it returns a null handle, so this
+ * function uses RTLD_NOW.
  * handle in this platform.
  * \p error points to MonoError where an error will be stored in
  * case of failure.   The error needs to be cleared when done using it, \c mono_error_cleanup.
@@ -195,7 +196,7 @@ mono_dl_open_self (MonoError *error)
 		return NULL;
 	}
 	mono_refcount_init (module, NULL);
-	module->handle = RTLD_DEFAULT;
+	module->handle = dlopen(NULL, RTLD_NOW);
 	module->dl_fallback = NULL;
 	module->full_name = NULL;
 	return module;

src/native/libs/System.Native/pal_dynamicload.c

@@ -56,24 +56,23 @@ void SystemNative_FreeLibrary(void* handle)
     dlclose(handle);
 }
 
-#ifdef TARGET_ANDROID
-void* SystemNative_GetDefaultSearchOrderPseudoHandle(void)
-{
-    return (void*)RTLD_DEFAULT;
-}
-#else
 static void* volatile g_defaultSearchOrderPseudoHandle = NULL;
 void* SystemNative_GetDefaultSearchOrderPseudoHandle(void)
 {
     // Read the value once from the volatile static to avoid reading from memory twice.
     void* defaultSearchOrderPseudoHandle = (void*)g_defaultSearchOrderPseudoHandle;
     if (defaultSearchOrderPseudoHandle == NULL)
     {
+#ifdef TARGET_ANDROID
+        int flag = RTLD_NOW;
+#else
+        int flag = RTLD_LAZY;
+#endif
+
         // Assign back to the static as well as the local here.
         // We don't need to check for a race between two threads as the value returned by
         // dlopen here will always be the same in a given environment.
-        g_defaultSearchOrderPseudoHandle = defaultSearchOrderPseudoHandle = dlopen(NULL, RTLD_LAZY);
+        g_defaultSearchOrderPseudoHandle = defaultSearchOrderPseudoHandle = dlopen(NULL, flag);
     }
     return defaultSearchOrderPseudoHandle;
 }
-#endif

src/tests/Interop/NativeLibrary/MainProgramHandle/MainProgramHandleTests.cs

@@ -0,0 +1,46 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using System.Runtime.InteropServices;
+
+using Xunit;
+
+public static class MainProgramHandleTests
+{
+    private static IntPtr s_handle;
+
+    static MainProgramHandleTests() => NativeLibrary.SetDllImportResolver(typeof(MainProgramHandleTests).Assembly,
+        (string libraryName, Assembly asm, DllImportSearchPath? dllImportSearchPath) =>
+        {
+            if (libraryName == "Self")
+            {
+                s_handle = NativeLibrary.GetMainProgramHandle();
+                Assert.NotEqual(IntPtr.Zero, s_handle);
+                return s_handle;
+            }
+
+            return IntPtr.Zero;
+        });
+
+    public static int Main()
+    {
+        try
+        {
+            free(s_handle);
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine($"Test Failure: {e}");
+            return 101;
+        }
+
+        return 100;
+    }
+
+    [DllImport("Self")]
+    private static extern void free(IntPtr arg);
+}

src/tests/Interop/NativeLibrary/MainProgramHandle/MainProgramHandleTests.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <CLRTestTargetUnsupported Condition="'$(TargetsWindows)' == 'true' or '$(RuntimeVariant)' == 'monointerpreter'">true</CLRTestTargetUnsupported>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="MainProgramHandleTests.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
+  </ItemGroup>
+</Project>

src/tests/issues.targets

@@ -2600,6 +2600,9 @@
         <ExcludeList Include = "$(XunitTestBinBase)/Interop/StructMarshalling/ReversePInvoke/MarshalSeqStruct/DelegatePInvoke/DelegatePInvokeTest/**">
             <Issue>needs triage</Issue>
         </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/Interop/NativeLibrary/MainProgramHandle/**">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
         <ExcludeList Include = "$(XunitTestBinBase)/JIT/Methodical/eh/basics/throwinfilter_il_d/**">
             <Issue>https://github.com/dotnet/runtime/issues/47624</Issue>
         </ExcludeList>