C#: Roslyn-based stub generation

csharp/CSharp.sln

@@ -1,5 +1,4 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
 VisualStudioVersion = 15.0.27130.2036
 MinimumVisualStudioVersion = 10.0.40219.1
@@ -15,6 +14,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.De
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Standalone", "extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj", "{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.StubGenerator", "extractor\Semmle.Extraction.CSharp.StubGenerator\Semmle.Extraction.CSharp.StubGenerator.csproj", "{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Util", "extractor\Semmle.Extraction.CSharp.Util\Semmle.Extraction.CSharp.Util.csproj", "{998A0D4C-8BFC-4513-A28D-4816AFB89882}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL.Driver", "extractor\Semmle.Extraction.CIL.Driver\Semmle.Extraction.CIL.Driver.csproj", "{EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Driver", "extractor\Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj", "{C36453BF-0C82-448A-B15D-26947503A2D3}"
@@ -29,6 +32,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp", "
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp.Tests", "autobuilder\Semmle.Autobuild.CSharp.Tests\Semmle.Autobuild.CSharp.Tests.csproj", "{34256E8F-866A-46C1-800E-3DF69FD1DCB7}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Extraction.CSharp.DependencyStubGenerator", "extractor\Semmle.Extraction.CSharp.DependencyStubGenerator\Semmle.Extraction.CSharp.DependencyStubGenerator.csproj", "{0EDA21A3-ADD8-4C10-B494-58B12B526B76}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -85,6 +90,18 @@ Global
 		{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Program.cs

@@ -0,0 +1,9 @@
+using Semmle.Extraction.CSharp.DependencyFetching;
+using Semmle.Extraction.CSharp.StubGenerator;
+using Semmle.Util.Logging;
+
+var logger = new ConsoleLogger(Verbosity.Info, logThreadId: false);
+using var dependencyManager = new DependencyManager(".", DependencyOptions.Default, logger);
+StubGenerator.GenerateStubs(logger, dependencyManager.ReferenceFiles, "codeql_csharp_stubs");
+
+return 0;

csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Semmle.Extraction.CSharp.DependencyStubGenerator.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <AssemblyName>Semmle.Extraction.CSharp.DependencyStubGenerator</AssemblyName>
+    <RootNamespace>Semmle.Extraction.CSharp.DependencyStubGenerator</RootNamespace>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.StubGenerator\Semmle.Extraction.CSharp.StubGenerator.csproj" />
+	</ItemGroup>
+
+</Project>

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/RelevantSymbol.cs

@@ -0,0 +1,33 @@
+using System.Linq;
+
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+
+using Semmle.Util;
+
+namespace Semmle.Extraction.CSharp.StubGenerator;
+
+internal class RelevantSymbol
+{
+    private readonly IAssemblySymbol assembly;
+    private readonly MemoizedFunc<INamedTypeSymbol, bool> isRelevantNamedType;
+    private readonly MemoizedFunc<INamespaceSymbol, bool> isRelevantNamespace;
+
+    public RelevantSymbol(IAssemblySymbol assembly)
+    {
+        this.assembly = assembly;
+
+        isRelevantNamedType = new(symbol =>
+            StubVisitor.IsRelevantBaseType(symbol) &&
+            SymbolEqualityComparer.Default.Equals(symbol.ContainingAssembly, assembly));
+
+        isRelevantNamespace = new(symbol =>
+            symbol.GetTypeMembers().Any(IsRelevantNamedType) ||
+            symbol.GetNamespaceMembers().Any(IsRelevantNamespace));
+    }
+
+    public bool IsRelevantNamedType(INamedTypeSymbol symbol) => isRelevantNamedType.Invoke(symbol);
+
+    public bool IsRelevantNamespace(INamespaceSymbol symbol) => isRelevantNamespace.Invoke(symbol);
+}
+

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/Semmle.Extraction.CSharp.StubGenerator.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+	<PropertyGroup>
+		<TargetFramework>net7.0</TargetFramework>
+		<AssemblyName>Semmle.Extraction.CSharp.StubGenerator</AssemblyName>
+		<RootNamespace>Semmle.Extraction.CSharp.StubGenerator</RootNamespace>
+		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+		<Nullable>enable</Nullable>
+	</PropertyGroup>
+	<ItemGroup>
+		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.Util\Semmle.Extraction.CSharp.Util.csproj" />
+	</ItemGroup>
+	<ItemGroup>
+		<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.4.0" />
+	</ItemGroup>
+</Project>

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubGenerator.cs

@@ -0,0 +1,81 @@
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.StubGenerator;
+
+public static class StubGenerator
+{
+    /// <summary>
+    /// Generates stubs for all the provided assembly paths.
+    /// </summary>
+    /// <param name="referencesPaths">The paths of the assemblies to generate stubs for.</param>
+    /// <param name="outputPath">The path in which to store the stubs.</param>
+    public static void GenerateStubs(ILogger logger, IEnumerable<string> referencesPaths, string outputPath)
+    {
+        var stopWatch = new System.Diagnostics.Stopwatch();
+        stopWatch.Start();
+
+        var threads = EnvironmentVariables.GetDefaultNumberOfThreads();
+
+        using var references = new BlockingCollection<(MetadataReference Reference, string Path)>();
+
+        Parallel.ForEach(referencesPaths, new ParallelOptions { MaxDegreeOfParallelism = threads }, path =>
+        {
+            var reference = MetadataReference.CreateFromFile(path);
+            references.Add((reference, path));
+        });
+
+        logger.Log(Severity.Info, $"Generating stubs for {references.Count} assemblies.");
+
+        var compilation = CSharpCompilation.Create(
+            "stubgenerator.dll",
+            null,
+            references.Select(tuple => tuple.Item1),
+            new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true));
+
+        Parallel.ForEach(references, new ParallelOptions { MaxDegreeOfParallelism = threads }, @ref =>
+        {
+            StubReference(logger, compilation, outputPath, @ref.Reference, @ref.Path);
+        });
+
+        stopWatch.Stop();
+        logger.Log(Severity.Info, $"Stub generation took {stopWatch.Elapsed}.");
+    }
+
+    private static void StubReference(ILogger logger, CSharpCompilation compilation, string outputPath, MetadataReference reference, string path)
+    {
+        if (compilation.GetAssemblyOrModuleSymbol(reference) is not IAssemblySymbol assembly)
+            return;
+
+        var relevantSymbol = new RelevantSymbol(assembly);
+
+        if (!assembly.Modules.Any(m => relevantSymbol.IsRelevantNamespace(m.GlobalNamespace)))
+            return;
+
+        using var fileStream = new FileStream(FileUtils.NestPaths(logger, outputPath, path.Replace(".dll", ".cs")), FileMode.Create, FileAccess.Write);
+        using var writer = new StreamWriter(fileStream, new UTF8Encoding(false));
+
+        var visitor = new StubVisitor(writer, relevantSymbol);
+
+        writer.WriteLine("// This file contains auto-generated code.");
+        writer.WriteLine($"// Generated from `{assembly.Identity}`.");
+
+        visitor.StubAttributes(assembly.GetAttributes(), "assembly: ");
+
+        foreach (var module in assembly.Modules)
+        {
+            module.GlobalNamespace.Accept(visitor);
+        }
+    }
+}
+