[DllImportGenerator] Don't flag methods with known unsupported UnmanagedType (#61808)

Author: elinor-fung

docs/design/libraries/DllImportGenerator/Compatibility.md

@@ -33,7 +33,7 @@ These are all part of `NetCoreApp` and will be referenced by default unless [imp
 
 Marshalling of `char` will not be supported when configured with any of the following:
   - [`CharSet.Ansi`, `CharSet.Auto`, or `CharSet.None`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.charset) will not be supported.
-  - [`UnmangedType.U1` or `UnmangedType.I1`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.unmanagedtype)
+  - [`UnmanagedType.U1` or `UnmanagedType.I1`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.unmanagedtype)
   - No explicit marshalling information - either [`DllImportAttribute.CharSet`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.dllimportattribute.charset) or [`MarshalAsAttribute`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.marshalasattribute)
 
 For `CharSet.Ansi` and `CharSet.None`, the built-in system used the [system default Windows ANSI code page](https://docs.microsoft.com/windows/win32/api/stringapiset/nf-stringapiset-widechartomultibyte) when on Windows and took the first byte of the UTF-8 encoding on non-Windows platforms. The above reasoning also applies to marshalling of a `char` as `UnmanagedType.U1` and `UnmanagedType.I1`. All approaches are fundamentally flawed and therefore not supported. If a single-byte character is expected to be marshalled it is left to the caller to convert a .NET `char` into a single `byte` prior to calling the native function.
@@ -44,7 +44,7 @@ For `CharSet.Auto`, the built-in system relied upon detection at runtime of the
 
 Marshalling of `string` will not be supported when configured with any of the following:
   - [`CharSet.None`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.charset)
-  - [`UnmangedType.VBByRefStr`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.unmanagedtype)
+  - [`UnmanagedType.VBByRefStr`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.unmanagedtype)
   - No explicit marshalling information - either [`DllImportAttribute.CharSet`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.dllimportattribute.charset) or [`MarshalAsAttribute`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.marshalasattribute)
 
 When converting from native to managed, the built-in system would throw a [`MarshalDirectiveException`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.marshaldirectiveexception) if the string's length is over 0x7ffffff0. The generated marshalling code will no longer perform this check.
@@ -63,7 +63,7 @@ Using a custom marshaller (i.e. [`ICustomMarshaler`](https://docs.microsoft.com/
 
 ### Array marshalling
 
-Marshalling of arrays will not be supported when using [`UnmangedType.SafeArray`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.unmanagedtype). This implies that the following `MarshalAsAttribute` fields are unsupported: [`SafeArraySubType`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.marshalasattribute.safearraysubtype) and [`SafeArrayUserDefinedSubType`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.marshalasattribute.safearrayuserdefinedsubtype)
+Marshalling of arrays will not be supported when using [`UnmanagedType.SafeArray`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.unmanagedtype). This implies that the following `MarshalAsAttribute` fields are unsupported: [`SafeArraySubType`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.marshalasattribute.safearraysubtype) and [`SafeArrayUserDefinedSubType`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.marshalasattribute.safearrayuserdefinedsubtype)
 
 Specifying array-specific marshalling members on the `MarshalAsAttribute` such as [`SizeConst`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.marshalasattribute.sizeconst), [`ArraySubType`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.marshalasattribute.arraysubtype), and [`SizeParamIndex`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.marshalasattribute.sizeparamindex) with non-array `UnmanagedType` types is unsupported.
 
@@ -96,6 +96,12 @@ Unlike the built-in system, the source generator does not support marshalling fo
 - [`HandleRef`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.handleref)
 - [`StringBuilder`](https://docs.microsoft.com/dotnet/api/system.text.stringbuilder)
 
+The source generator also does not support marshalling objects using the following [`UnmanagedType`](https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.unmanagedtype) values:
+- `UnmanagedType.Interface`
+- `UnmanagedType.IDispatch`
+- `UnmanagedType.IInspectable`
+- `UnmanagedType.IUnknown`
+
 ## Version 0
 
 This version is the built-in IL Stub generation system that is triggered whenever a method marked with `DllImportAttribute` is invoked.

src/libraries/System.Runtime.InteropServices/gen/DllImportGenerator/Analyzers/ConvertToGeneratedDllImportAnalyzer.cs

@@ -3,6 +3,7 @@
 
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Runtime.InteropServices;
 
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.Diagnostics;
@@ -48,6 +49,8 @@ public override void Initialize(AnalysisContext context)
                     if (generatedDllImportAttrType == null)
                         return;
 
+                    INamedTypeSymbol? marshalAsAttrType = compilationContext.Compilation.GetTypeByMetadataName(TypeNames.System_Runtime_InteropServices_MarshalAsAttribute);
+
                     var knownUnsupportedTypes = new List<ITypeSymbol>(s_unsupportedTypeNames.Length);
                     foreach (string typeName in s_unsupportedTypeNames)
                     {
@@ -58,11 +61,11 @@ public override void Initialize(AnalysisContext context)
                         }
                     }
 
-                    compilationContext.RegisterSymbolAction(symbolContext => AnalyzeSymbol(symbolContext, knownUnsupportedTypes), SymbolKind.Method);
+                    compilationContext.RegisterSymbolAction(symbolContext => AnalyzeSymbol(symbolContext, knownUnsupportedTypes, marshalAsAttrType), SymbolKind.Method);
                 });
         }
 
-        private static void AnalyzeSymbol(SymbolAnalysisContext context, List<ITypeSymbol> knownUnsupportedTypes)
+        private static void AnalyzeSymbol(SymbolAnalysisContext context, List<ITypeSymbol> knownUnsupportedTypes, INamedTypeSymbol? marshalAsAttrType)
         {
             var method = (IMethodSymbol)context.Symbol;
 
@@ -84,17 +87,53 @@ private static void AnalyzeSymbol(SymbolAnalysisContext context, List<ITypeSymbo
             // Ignore methods with unsupported parameters
             foreach (IParameterSymbol parameter in method.Parameters)
             {
-                if (knownUnsupportedTypes.Contains(parameter.Type))
+                if (knownUnsupportedTypes.Contains(parameter.Type)
+                    || HasUnsupportedUnmanagedTypeValue(parameter.GetAttributes(), marshalAsAttrType))
                 {
                     return;
                 }
             }
 
             // Ignore methods with unsupported returns
-            if (method.ReturnsByRef || method.ReturnsByRefReadonly || knownUnsupportedTypes.Contains(method.ReturnType))
+            if (method.ReturnsByRef || method.ReturnsByRefReadonly)
+                return;
+
+            if (knownUnsupportedTypes.Contains(method.ReturnType) || HasUnsupportedUnmanagedTypeValue(method.GetReturnTypeAttributes(), marshalAsAttrType))
                 return;
 
             context.ReportDiagnostic(method.CreateDiagnostic(ConvertToGeneratedDllImport, method.Name));
         }
+
+        private static bool HasUnsupportedUnmanagedTypeValue(ImmutableArray<AttributeData> attributes, INamedTypeSymbol? marshalAsAttrType)
+        {
+            if (marshalAsAttrType == null)
+                return false;
+
+            AttributeData? marshalAsAttr = null;
+            foreach (AttributeData attr in attributes)
+            {
+                if (SymbolEqualityComparer.Default.Equals(attr.AttributeClass, marshalAsAttrType))
+                {
+                    marshalAsAttr = attr;
+                    break;
+                }
+            }
+
+            if (marshalAsAttr == null || marshalAsAttr.ConstructorArguments.IsEmpty)
+                return false;
+
+            object unmanagedTypeObj = marshalAsAttr.ConstructorArguments[0].Value!;
+            UnmanagedType unmanagedType = unmanagedTypeObj is short unmanagedTypeAsShort
+                ? (UnmanagedType)unmanagedTypeAsShort
+                : (UnmanagedType)unmanagedTypeObj;
+
+            return !System.Enum.IsDefined(typeof(UnmanagedType), unmanagedType)
+                || unmanagedType == UnmanagedType.CustomMarshaler
+                || unmanagedType == UnmanagedType.Interface
+                || unmanagedType == UnmanagedType.IDispatch
+                || unmanagedType == UnmanagedType.IInspectable
+                || unmanagedType == UnmanagedType.IUnknown
+                || unmanagedType == UnmanagedType.SafeArray;
+        }
     }
 }

src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/ConvertToGeneratedDllImportAnalyzerTests.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 
 using Microsoft.CodeAnalysis;
@@ -153,6 +154,29 @@ public async Task UnsupportedType_NoDiagnostic(Type type)
             await VerifyCS.VerifyAnalyzerAsync(source);
         }
 
+        [ConditionalTheory]
+        [InlineData(UnmanagedType.Interface)]
+        [InlineData(UnmanagedType.IDispatch)]
+        [InlineData(UnmanagedType.IInspectable)]
+        [InlineData(UnmanagedType.IUnknown)]
+        [InlineData(UnmanagedType.SafeArray)]
+        public async Task UnsupportedUnmanagedType_NoDiagnostic(UnmanagedType unmanagedType)
+        {
+            string source = $@"
+using System.Runtime.InteropServices;
+unsafe partial class Test
+{{
+    [DllImport(""DoesNotExist"")]
+    public static extern void Method_Parameter([MarshalAs(UnmanagedType.{unmanagedType}, MarshalType = ""DNE"")]int p);
+
+    [DllImport(""DoesNotExist"")]
+    [return: MarshalAs(UnmanagedType.{unmanagedType}, MarshalType = ""DNE"")]
+    public static extern int Method_Return();
+}}
+";
+            await VerifyCS.VerifyAnalyzerAsync(source);
+        }
+
         [ConditionalFact]
         public async Task GeneratedDllImport_NoDiagnostic()
         {