C#: Roslyn-based stub generation

csharp/CSharp.sln

@@ -14,6 +14,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.De
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Standalone", "extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj", "{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.StubGenerator", "extractor\Semmle.Extraction.CSharp.StubGenerator\Semmle.Extraction.CSharp.StubGenerator.csproj", "{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Util", "extractor\Semmle.Extraction.CSharp.Util\Semmle.Extraction.CSharp.Util.csproj", "{998A0D4C-8BFC-4513-A28D-4816AFB89882}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL.Driver", "extractor\Semmle.Extraction.CIL.Driver\Semmle.Extraction.CIL.Driver.csproj", "{EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}"
@@ -30,6 +32,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp", "
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp.Tests", "autobuilder\Semmle.Autobuild.CSharp.Tests\Semmle.Autobuild.CSharp.Tests.csproj", "{34256E8F-866A-46C1-800E-3DF69FD1DCB7}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Extraction.CSharp.DependencyStubGenerator", "extractor\Semmle.Extraction.CSharp.DependencyStubGenerator\Semmle.Extraction.CSharp.DependencyStubGenerator.csproj", "{0EDA21A3-ADD8-4C10-B494-58B12B526B76}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -90,6 +94,14 @@ Global
 		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Program.cs

@@ -0,0 +1,9 @@
+using Semmle.Extraction.CSharp.DependencyFetching;
+using Semmle.Extraction.CSharp.StubGenerator;
+using Semmle.Util.Logging;
+
+var logger = new ConsoleLogger(Verbosity.Info);
+using var dependencyManager = new DependencyManager(".", DependencyOptions.Default, logger);
+StubGenerator.GenerateStubs(logger, dependencyManager.ReferenceFiles, "codeql_csharp_stubs");
+
+return 0;

csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Semmle.Extraction.CSharp.DependencyStubGenerator.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <AssemblyName>Semmle.Extraction.CSharp.DependencyStubGenerator</AssemblyName>
+    <RootNamespace>Semmle.Extraction.CSharp.DependencyStubGenerator</RootNamespace>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.StubGenerator\Semmle.Extraction.CSharp.StubGenerator.csproj" />
+	</ItemGroup>
+
+</Project>

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/Semmle.Extraction.CSharp.StubGenerator.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+	<PropertyGroup>
+		<TargetFramework>net7.0</TargetFramework>
+		<AssemblyName>Semmle.Extraction.CSharp.StubGenerator</AssemblyName>
+		<RootNamespace>Semmle.Extraction.CSharp.StubGenerator</RootNamespace>
+		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+		<Nullable>enable</Nullable>
+	</PropertyGroup>
+	<ItemGroup>
+		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
+		<ProjectReference Include="..\Semmle.Extraction.CSharp.Util\Semmle.Extraction.CSharp.Util.csproj" />
+	</ItemGroup>
+	<ItemGroup>
+		<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.4.0" />
+	</ItemGroup>
+</Project>

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubGenerator.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.StubGenerator;
+
+public static class StubGenerator
+{
+    /// <summary>
+    /// Generates stubs for all the provided assembly paths.
+    /// </summary>
+    /// <param name="referencesPaths">The paths of the assemblies to generate stubs for.</param>
+    /// <param name="outputPath">The path in which to store the stubs.</param>
+    public static void GenerateStubs(ILogger logger, IEnumerable<string> referencesPaths, string outputPath)
+    {
+        var stopWatch = new System.Diagnostics.Stopwatch();
+        stopWatch.Start();
+
+        var threads = EnvironmentVariables.GetDefaultNumberOfThreads();
+
+        using var references = new BlockingCollection<(MetadataReference Reference, string Path)>();
+        var referenceResolveTasks = GetResolvedReferenceTasks(referencesPaths, references);
+
+        Parallel.Invoke(
+            new ParallelOptions { MaxDegreeOfParallelism = threads },
+            referenceResolveTasks.ToArray());
+
+        logger.Log(Severity.Info, $"Generating stubs for {references.Count} assemblies.");
+
+        var compilation = CSharpCompilation.Create(
+            "stubgenerator.dll",
+            null,
+            references.Select(tuple => tuple.Item1),
+            new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true));
+
+        var referenceStubTasks = references.Select(@ref => (Action)(() => StubReference(compilation, outputPath, @ref.Reference, @ref.Path)));
+        Parallel.Invoke(
+            new ParallelOptions { MaxDegreeOfParallelism = threads },
+            referenceStubTasks.ToArray());
+
+        stopWatch.Stop();
+        logger.Log(Severity.Info, $"Stub generation took {stopWatch.Elapsed}.");
+    }
+
+    private static IEnumerable<Action> GetResolvedReferenceTasks(IEnumerable<string> referencePaths, BlockingCollection<(MetadataReference, string)> references)
+    {
+        return referencePaths.Select<string, Action>(path => () =>
+        {
+            var reference = MetadataReference.CreateFromFile(path);
+            references.Add((reference, path));
+        });
+    }
+
+    private static void StubReference(CSharpCompilation compilation, string outputPath, MetadataReference reference, string path)
+    {
+        if (compilation.GetAssemblyOrModuleSymbol(reference) is IAssemblySymbol assembly)
+        {
+            var logger = new ConsoleLogger(Verbosity.Info);
+            using var fileStream = new FileStream(FileUtils.NestPaths(logger, outputPath, path.Replace(".dll", ".cs")), FileMode.Create, FileAccess.Write);
+            using var writer = new StreamWriter(fileStream, new UTF8Encoding(false));
+
+            writer.WriteLine("// This file contains auto-generated code.");
+            writer.WriteLine($"// Generated from `{assembly.Identity}`.");
+
+            var visitor = new StubVisitor(assembly, writer);
+
+            visitor.StubAttributes(assembly.GetAttributes(), "assembly: ");
+
+            foreach (var module in assembly.Modules)
+            {
+                module.GlobalNamespace.Accept(new StubVisitor(assembly, writer));
+            }
+        }
+    }
+}
+