Use Roslyn to parse directives in file-based programs (#47978)

Author: jjonescz

src/Cli/dotnet/Commands/CliCommandStrings.resx

@@ -367,6 +367,9 @@ For MSTest before 2.2.4, the timeout is used for all testcases.</value>
   <data name="CmdFileDescription" xml:space="preserve">
     <value>Path to the file-based program.</value>
   </data>
+  <data name="CmdOptionForceDescription" xml:space="preserve">
+    <value>Force conversion even if there are malformed directives.</value>
+  </data>
   <data name="CmdForceRestoreOptionDescription" xml:space="preserve">
     <value>Force all dependencies to be resolved even if the last restore was successful.
 This is equivalent to deleting project.assets.json.</value>
@@ -1493,8 +1496,12 @@ Tool '{1}' (version '{2}') was successfully installed. Entry is added to the man
     <comment>{0} is the file path and line number. {1} is an inner exception message.</comment>
   </data>
   <data name="PropertyDirectiveMissingParts" xml:space="preserve">
-    <value>The property directive needs to have two parts separated by '=' like 'PropertyName=PropertyValue': {0}</value>
-    <comment>{0} is the file path and line number. {Locked="="}</comment>
+    <value>The property directive needs to have two parts separated by a space like 'PropertyName PropertyValue': {0}</value>
+    <comment>{0} is the file path and line number.</comment>
+  </data>
+  <data name="CannotConvertDirective" xml:space="preserve">
+    <value>Some directives cannot be converted: the first error is at {0}. Run the file to see all compilation errors. Specify '--force' to convert anyway.</value>
+    <comment>{Locked="--force"}. {0} is the file path and line number.</comment>
   </data>
   <data name="PublishAppDescription" xml:space="preserve">
     <value>Publisher for the .NET Platform</value>

src/Cli/dotnet/Commands/Project/Convert/ProjectConvertCommand.cs

@@ -14,6 +14,7 @@ internal sealed class ProjectConvertCommand(ParseResult parseResult) : CommandBa
 {
     private readonly string _file = parseResult.GetValue(ProjectConvertCommandParser.FileArgument) ?? string.Empty;
     private readonly string? _outputDirectory = parseResult.GetValue(SharedOptions.OutputOption)?.FullName;
+    private readonly bool _force = parseResult.GetValue(ProjectConvertCommandParser.ForceOption);
 
     public override int Execute()
     {
@@ -31,7 +32,7 @@ public override int Execute()
 
         // Find directives (this can fail, so do this before creating the target directory).
         var sourceFile = VirtualProjectBuildingCommand.LoadSourceFile(file);
-        var directives = VirtualProjectBuildingCommand.FindDirectives(sourceFile);
+        var directives = VirtualProjectBuildingCommand.FindDirectivesForConversion(sourceFile, force: _force);
 
         Directory.CreateDirectory(targetDirectory);
 

src/Cli/dotnet/Commands/Project/Convert/ProjectConvertCommandParser.cs

@@ -16,12 +16,19 @@ internal sealed class ProjectConvertCommandParser
         Arity = ArgumentArity.ExactlyOne,
     };
 
+    public static readonly CliOption<bool> ForceOption = new("--force")
+    {
+        Description = CliCommandStrings.CmdOptionForceDescription,
+        Arity = ArgumentArity.Zero,
+    };
+
     public static CliCommand GetCommand()
     {
         CliCommand command = new("convert", CliCommandStrings.ProjectConvertAppFullName)
         {
             FileArgument,
             SharedOptions.OutputOption,
+            ForceOption,
         };
 
         command.SetAction((parseResult) => new ProjectConvertCommand(parseResult).Execute());

src/Cli/dotnet/Commands/Run/VirtualProjectBuildingCommand.cs

@@ -15,6 +15,8 @@
 using Microsoft.Build.Framework;
 using Microsoft.Build.Logging;
 using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Microsoft.CodeAnalysis.Text;
 using Microsoft.DotNet.Cli.Utils;
 
@@ -26,7 +28,6 @@ namespace Microsoft.DotNet.Cli.Commands.Run;
 internal sealed class VirtualProjectBuildingCommand
 {
     private ImmutableArray<CSharpDirective> _directives;
-    private string? _targetFilePath;
 
     public Dictionary<string, string> GlobalProperties { get; } = new(StringComparer.OrdinalIgnoreCase);
     public required string EntryPointFileFullPath { get; init; }
@@ -132,22 +133,10 @@ public int Execute(string[] binaryLoggerArgs, ILogger consoleLogger)
     /// </summary>
     public VirtualProjectBuildingCommand PrepareProjectInstance()
     {
-        Debug.Assert(_directives.IsDefault && _targetFilePath is null, $"{nameof(PrepareProjectInstance)} should not be called multiple times.");
+        Debug.Assert(_directives.IsDefault, $"{nameof(PrepareProjectInstance)} should not be called multiple times.");
 
         var sourceFile = LoadSourceFile(EntryPointFileFullPath);
-        _directives = FindDirectives(sourceFile);
-
-        // If there were any `#:` directives, remove them from the file.
-        // (This is temporary until Roslyn is updated to ignore them.)
-        _targetFilePath = EntryPointFileFullPath;
-        if (_directives.Length != 0)
-        {
-            var targetDirectory = Path.Join(Path.GetDirectoryName(_targetFilePath), "obj");
-            Directory.CreateDirectory(targetDirectory);
-            _targetFilePath = Path.Join(targetDirectory, Path.GetFileName(_targetFilePath));
-
-            RemoveDirectivesFromFile(_directives, sourceFile.Text, _targetFilePath);
-        }
+        _directives = FindDirectives(sourceFile, reportErrors: false);
 
         return this;
     }
@@ -177,15 +166,15 @@ private ProjectInstance CreateProjectInstance(
 
         ProjectRootElement CreateProjectRootElement(ProjectCollection projectCollection)
         {
-            Debug.Assert(!_directives.IsDefault && _targetFilePath is not null, $"{nameof(PrepareProjectInstance)} should have been called first.");
+            Debug.Assert(!_directives.IsDefault, $"{nameof(PrepareProjectInstance)} should have been called first.");
 
             var projectFileFullPath = Path.ChangeExtension(EntryPointFileFullPath, ".csproj");
             var projectFileWriter = new StringWriter();
             WriteProjectFile(
                 projectFileWriter,
                 _directives,
                 isVirtualProject: true,
-                targetFilePath: _targetFilePath,
+                targetFilePath: EntryPointFileFullPath,
                 artifactsPath: GetArtifactsPath(EntryPointFileFullPath));
             var projectFileText = projectFileWriter.ToString();
 
@@ -330,6 +319,17 @@ private static void WriteProjectFile(
             writer.WriteLine("  </PropertyGroup>");
         }
 
+        if (isVirtualProject)
+        {
+            // After `#:property` directives so they don't override this.
+            writer.WriteLine("""
+
+                  <PropertyGroup>
+                    <Features>$(Features);FileBasedProgram</Features>
+                  </PropertyGroup>
+                """);
+        }
+
         if (packageDirectives.Any())
         {
             writer.WriteLine("""
@@ -426,28 +426,101 @@ Override targets which don't work with project files that are not present on dis
         static string EscapeValue(string value) => SecurityElement.Escape(value);
     }
 
-    public static ImmutableArray<CSharpDirective> FindDirectives(SourceFile sourceFile)
+    public static ImmutableArray<CSharpDirective> FindDirectivesForConversion(SourceFile sourceFile, bool force)
+    {
+        return FindDirectives(sourceFile, reportErrors: !force);
+    }
+
+#pragma warning disable RSEXPERIMENTAL003 // 'SyntaxTokenParser' is experimental
+#pragma warning disable RSEXPERIMENTAL005 // 'IgnoredDirectiveTriviaSyntax' is experimental
+    private static ImmutableArray<CSharpDirective> FindDirectives(SourceFile sourceFile, bool reportErrors)
     {
         var builder = ImmutableArray.CreateBuilder<CSharpDirective>();
+        SyntaxTokenParser tokenizer = SyntaxFactory.CreateTokenParser(sourceFile.Text,
+            CSharpParseOptions.Default.WithFeatures([new("FileBasedProgram", "true")]));
 
-        // NOTE: When Roslyn is updated to support "ignored directives", we should use its SyntaxTokenParser instead.
-        foreach (var line in sourceFile.Text.Lines)
+        var result = tokenizer.ParseLeadingTrivia();
+        TextSpan previousWhiteSpaceSpan = default;
+        foreach (var trivia in result.Token.LeadingTrivia)
         {
-            var lineText = sourceFile.Text.ToString(line.Span);
+            // Stop when the trivia contains an error (e.g., because it's after #if).
+            if (trivia.ContainsDiagnostics)
+            {
+                break;
+            }
+
+            if (trivia.IsKind(SyntaxKind.WhitespaceTrivia))
+            {
+                Debug.Assert(previousWhiteSpaceSpan.IsEmpty);
+                previousWhiteSpaceSpan = trivia.FullSpan;
+                continue;
+            }
+
+            if (trivia.IsKind(SyntaxKind.ShebangDirectiveTrivia))
+            {
+                TextSpan span = getFullSpan(previousWhiteSpaceSpan, trivia);
 
-            if (Patterns.Shebang.IsMatch(lineText))
+                builder.Add(new CSharpDirective.Shebang { Span = span });
+            }
+            else if (trivia.IsKind(SyntaxKind.IgnoredDirectiveTrivia))
             {
-                builder.Add(new CSharpDirective.Shebang { Span = line.SpanIncludingLineBreak });
+                TextSpan span = getFullSpan(previousWhiteSpaceSpan, trivia);
+
+                var message = trivia.GetStructure() is IgnoredDirectiveTriviaSyntax { EndOfDirectiveToken.LeadingTrivia: [{ RawKind: (int)SyntaxKind.PreprocessingMessageTrivia } messageTrivia] }
+                    ? messageTrivia.ToString().AsSpan().Trim()
+                    : "";
+                var parts = Patterns.Whitespace.EnumerateSplits(message, 2);
+                var name = parts.MoveNext() ? message[parts.Current] : default;
+                var value = parts.MoveNext() ? message[parts.Current] : default;
+                Debug.Assert(!parts.MoveNext());
+                builder.Add(CSharpDirective.Parse(sourceFile, span, name.ToString(), value.ToString()));
             }
-            else if (Patterns.Directive.Match(lineText) is { Success: true } match)
+
+            previousWhiteSpaceSpan = default;
+        }
+
+        // In conversion mode, we want to report errors for any invalid directives in the rest of the file
+        // so users don't end up with invalid directives in the converted project.
+        if (reportErrors)
+        {
+            tokenizer.ResetTo(result);
+
+            do
             {
-                builder.Add(CSharpDirective.Parse(sourceFile, line.SpanIncludingLineBreak, match.Groups[1].Value, match.Groups[2].Value));
+                result = tokenizer.ParseNextToken();
+
+                foreach (var trivia in result.Token.LeadingTrivia)
+                {
+                    reportErrorFor(sourceFile, trivia);
+                }
+
+                foreach (var trivia in result.Token.TrailingTrivia)
+                {
+                    reportErrorFor(sourceFile, trivia);
+                }
             }
+            while (!result.Token.IsKind(SyntaxKind.EndOfFileToken));
         }
 
         // The result should be ordered by source location, RemoveDirectivesFromFile depends on that.
         return builder.ToImmutable();
+
+        static TextSpan getFullSpan(TextSpan previousWhiteSpaceSpan, SyntaxTrivia trivia)
+        {
+            // Include the preceding whitespace in the span, i.e., span will be the whole line.
+            return previousWhiteSpaceSpan.IsEmpty ? trivia.FullSpan : TextSpan.FromBounds(previousWhiteSpaceSpan.Start, trivia.FullSpan.End);
+        }
+
+        static void reportErrorFor(SourceFile sourceFile, SyntaxTrivia trivia)
+        {
+            if (trivia.ContainsDiagnostics && trivia.IsKind(SyntaxKind.IgnoredDirectiveTrivia))
+            {
+                throw new GracefulException(CliCommandStrings.CannotConvertDirective, sourceFile.GetLocationString(trivia.Span));
+            }
+        }
     }
+#pragma warning restore RSEXPERIMENTAL005 // 'IgnoredDirectiveTriviaSyntax' is experimental
+#pragma warning restore RSEXPERIMENTAL003 // 'SyntaxTokenParser' is experimental
 
     public static SourceFile LoadSourceFile(string filePath)
     {
@@ -500,11 +573,8 @@ public string GetLocationString(TextSpan span)
 
 internal static partial class Patterns
 {
-    [GeneratedRegex("""^\s*#:\s*(\w*)\s*(.*?)\s*$""")]
-    public static partial Regex Directive { get; }
-
-    [GeneratedRegex("""^\s*#!.*$""")]
-    public static partial Regex Shebang { get; }
+    [GeneratedRegex("""\s+""")]
+    public static partial Regex Whitespace { get; }
 }
 
 /// <summary>