File-based programs IDE support

Author: RikkiGibson

.vscode/tasks.json

@@ -169,7 +169,7 @@
       "type": "process",
       "options": {
         "env": {
-          "DOTNET_ROSLYN_SERVER_PATH": "${workspaceRoot}/artifacts/bin/Microsoft.CodeAnalysis.LanguageServer/Debug/net8.0/Microsoft.CodeAnalysis.LanguageServer.dll"
+          "DOTNET_ROSLYN_SERVER_PATH": "${workspaceRoot}/artifacts/bin/Microsoft.CodeAnalysis.LanguageServer/Debug/net9.0/Microsoft.CodeAnalysis.LanguageServer.dll"
         }
       },
       "dependsOn": [ "build language server" ]

docs/features/file-based-programs-vscode.md

@@ -0,0 +1,68 @@
+# File-based programs VS Code support
+
+See also [dotnet-run-file.md](https://github.com/dotnet/sdk/blob/main/documentation/general/dotnet-run-file.md).
+
+## Feature overview
+
+A file-based program embeds a subset of MSBuild project capabilities into C# code, allowing single files to stand alone as ordinary projects.
+
+The following is a file-based program:
+
+```cs
+Console.WriteLine("Hello World!");
+```
+
+So is the following:
+
+```cs
+#!/usr/bin/env dotnet run
+#:sdk Microsoft.Net.Sdk
+#:package Newtonsoft.Json@13.0.3
+#:property LangVersion=preview
+
+using Newtonsoft.Json;
+
+Main();
+
+void Main()
+{
+    if (args is not [_, var jsonPath, ..])
+    {
+        Console.Error.WriteLine("Usage: app <json-file>");
+        return;
+    }
+
+    var json = File.ReadAllText(jsonPath);
+    var data = JsonConvert.DeserializeObject<Data>(json);
+    // ...
+}
+
+record Data(string field1, int field2);
+```
+
+This basically works by having the `dotnet` command line interpret the `#:` directives found at the top (and only at the top) of source files, produce a C# project XML document in memory, and pass it off to MSBuild. The in-memory project is sometimes called a "virtual project".
+
+## Miscellaneous files changes
+
+There is a long-standing backlog item to enhance the experience of working with miscellaneous files ("loose files" not associated with any project). We think that as part of the "file-based program" work, we can enable the following in such files without substantial issues:
+- Syntax diagnostics.
+- Intellisense for the "default" set of references. e.g. those references which are included in the project created by `dotnet new console` with the current SDK.
+
+## High-level IDE layer changes
+
+The following outline is for "single-file" scenarios. We are interested in expanding to multi-file as well, and this document will be expanded in the future with additional explanation of how to handle that scenario.
+
+### Prerequisite work
+- MSBuildHost is updated to allow passing project content along with a file path. (https://github.com/dotnet/roslyn/pull/78303)
+- LanguageServerProjectSystem has a base type extracted so that a FileBasedProgramProjectSystem can be added. The latter handles loading file-based program projects, and makes the appropriate call to obtain the file-based program project XML. (https://github.com/dotnet/roslyn/pull/78329)
+
+### Heuristic
+The IDE considers a file to be a file-based program, if:
+- It has a `#!` directive such as `#!/usr/bin/env dotnet run`, or,
+- It has any `#:` directives which configure the file-based program project, or,
+- It has any top-level statements.
+- Any of the above is met, and, the file is not included in an ordinary `.csproj` project (i.e. it is not part of any ordinary project's list of `Compile` items).
+
+### Opt-out
+
+Before we ship this feature in any stable release, we want to ensure an opt-out flag is in place. Keeping the existing `LspMiscellaneousFilesWorkspace` may be a convenient point of control for this--if user has enabled the opt-out flag, just use the old workspace. Otherwise, use the new one.

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer.UnitTests/WorkspaceProjectFactoryServiceTests.cs

@@ -48,7 +48,7 @@ public async Task CreateProjectAndBatch()
         await batch.ApplyAsync(CancellationToken.None);
 
         // Verify it actually did something; we won't exclusively test each method since those are tested at lower layers
-        var project = workspaceFactory.Workspace.CurrentSolution.Projects.Single();
+        var project = workspaceFactory.HostWorkspace.CurrentSolution.Projects.Single();
 
         var document = Assert.Single(project.Documents);
         Assert.Equal(sourceFilePath, document.FilePath);

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer/HostWorkspace/FileBasedProgramsProjectSystem.cs

@@ -0,0 +1,139 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Immutable;
+using System.Security;
+using Microsoft.CodeAnalysis.LanguageServer.HostWorkspace.ProjectTelemetry;
+using Microsoft.CodeAnalysis.MetadataAsSource;
+using Microsoft.CodeAnalysis.MSBuild;
+using Microsoft.CodeAnalysis.Options;
+using Microsoft.CodeAnalysis.ProjectSystem;
+using Microsoft.CodeAnalysis.Shared.Extensions;
+using Microsoft.CodeAnalysis.Shared.TestHooks;
+using Microsoft.CodeAnalysis.Text;
+using Microsoft.CodeAnalysis.Workspaces.ProjectSystem;
+using Microsoft.CommonLanguageServerProtocol.Framework;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.Composition;
+using Roslyn.Utilities;
+using static Microsoft.CodeAnalysis.MSBuild.BuildHostProcessManager;
+
+namespace Microsoft.CodeAnalysis.LanguageServer.HostWorkspace;
+
+internal sealed class FileBasedProgramsProjectSystem : LanguageServerProjectLoader, ILspMiscellaneousFilesWorkspaceProvider
+{
+    private readonly ILspServices _lspServices;
+    private readonly ILogger<FileBasedProgramsProjectSystem> _logger;
+    private readonly IMetadataAsSourceFileService _metadataAsSourceFileService;
+
+    public FileBasedProgramsProjectSystem(
+        ILspServices lspServices,
+        IMetadataAsSourceFileService metadataAsSourceFileService,
+        LanguageServerWorkspaceFactory workspaceFactory,
+        IFileChangeWatcher fileChangeWatcher,
+        IGlobalOptionService globalOptionService,
+        ILoggerFactory loggerFactory,
+        IAsynchronousOperationListenerProvider listenerProvider,
+        ProjectLoadTelemetryReporter projectLoadTelemetry,
+        ServerConfigurationFactory serverConfigurationFactory,
+        BinlogNamer binlogNamer)
+            : base(
+                workspaceFactory.FileBasedProgramsProjectFactory,
+                workspaceFactory.TargetFrameworkManager,
+                workspaceFactory.ProjectSystemHostInfo,
+                fileChangeWatcher,
+                globalOptionService,
+                loggerFactory,
+                listenerProvider,
+                projectLoadTelemetry,
+                serverConfigurationFactory,
+                binlogNamer)
+    {
+        _lspServices = lspServices;
+        _logger = loggerFactory.CreateLogger<FileBasedProgramsProjectSystem>();
+        _metadataAsSourceFileService = metadataAsSourceFileService;
+    }
+
+    public Workspace Workspace => ProjectFactory.Workspace;
+
+    public async Task<TextDocument?> AddMiscellaneousDocumentAsync(Uri uri, SourceText documentText, string languageId, ILspLogger logger)
+    {
+        var documentPath = ProtocolConversions.GetDocumentFilePathFromUri(uri);
+        var container = documentText.Container;
+        if (_metadataAsSourceFileService.TryAddDocumentToWorkspace(documentPath, container, out var documentId))
+        {
+            var metadataWorkspace = _metadataAsSourceFileService.TryGetWorkspace();
+            Contract.ThrowIfNull(metadataWorkspace);
+            var metadataDoc = metadataWorkspace.CurrentSolution.GetRequiredDocument(documentId);
+            return metadataDoc;
+        }
+
+        var languageInfoProvider = _lspServices.GetRequiredService<ILanguageInfoProvider>();
+        if (!languageInfoProvider.TryGetLanguageInformation(uri, languageId, out var languageInformation))
+        {
+            // Only log here since throwing here could take down the LSP server.
+            logger.LogError($"Could not find language information for {uri} with absolute path {documentPath}");
+            return null;
+        }
+
+        // For Razor files we need to override the language name to C# as thats what code is generated
+        var isRazor = languageInformation.LanguageName == "Razor";
+        var languageName = isRazor ? LanguageNames.CSharp : languageInformation.LanguageName;
+        var loadedProject = await CreateAndTrackInitialProjectAsync(documentPath, language: languageName);
+        var documentFileInfo = new DocumentFileInfo(documentPath, logicalPath: documentPath, isLinked: false, isGenerated: false, folders: default);
+        var projectFileInfo = new ProjectFileInfo()
+        {
+            Language = languageName,
+            FilePath = uri.IsFile ? VirtualProject.GetVirtualProjectPath(documentPath) : documentPath,
+            CommandLineArgs = uri.IsFile ? ["/langversion:preview", "/features:FileBasedProgram=true"] : ["/langversion:preview"],
+            Documents = isRazor ? [] : [documentFileInfo],
+            AdditionalDocuments = isRazor ? [documentFileInfo] : [],
+            AnalyzerConfigDocuments = [],
+            ProjectReferences = [],
+            PackageReferences = [],
+            ProjectCapabilities = [],
+            ContentFilePaths = [],
+            FileGlobs = []
+        };
+        await loadedProject.UpdateWithNewProjectInfoAsync(projectFileInfo, _logger);
+        var workspaceProject = ProjectFactory.Workspace.CurrentSolution.GetRequiredProject(loadedProject.ProjectId);
+        var document = isRazor ? workspaceProject.AdditionalDocuments.Single() : workspaceProject.Documents.Single();
+
+        if (uri.IsFile && languageInformation.LanguageName == LanguageNames.CSharp)
+        {
+            // light up the proper file-based program experience.
+            ProjectsToLoadAndReload.AddWork(new ProjectToLoad(documentPath, ProjectGuid: null, ReportTelemetry: true));
+            loadedProject.NeedsReload += (_, _) => ProjectsToLoadAndReload.AddWork(new ProjectToLoad(documentPath, ProjectGuid: null, ReportTelemetry: false));
+
+            _ = Task.Run(async () =>
+            {
+                await ProjectsToLoadAndReload.WaitUntilCurrentBatchCompletesAsync();
+                await ProjectInitializationHandler.SendProjectInitializationCompleteNotificationAsync();
+            });
+        }
+
+        Contract.ThrowIfFalse(document.FilePath == documentPath);
+        return document;
+    }
+
+    public void TryRemoveMiscellaneousDocument(Uri uri, bool removeFromMetadataWorkspace)
+    {
+        // support unloading
+    }
+
+    protected override async Task<(RemoteProjectFile? projectFile, BuildHostProcessKind preferred, BuildHostProcessKind actual)> TryLoadProjectAsync(
+        BuildHostProcessManager buildHostProcessManager, ProjectToLoad projectToLoad, CancellationToken cancellationToken)
+    {
+        const BuildHostProcessKind buildHostKind = BuildHostProcessKind.NetCore;
+        var buildHost = await buildHostProcessManager.GetBuildHostAsync(buildHostKind, cancellationToken);
+        var documentPath = projectToLoad.Path;
+        Contract.ThrowIfFalse(Path.GetExtension(documentPath) == ".cs");
+
+        var fakeProjectPath = VirtualProject.GetVirtualProjectPath(documentPath);
+        var contentToLoad = VirtualProject.MakeVirtualProjectContent(documentPath);
+
+        var loadedFile = await buildHost.LoadProjectAsync(fakeProjectPath, contentToLoad, languageName: LanguageNames.CSharp, cancellationToken);
+        return (loadedFile, preferred: buildHostKind, actual: buildHostKind);
+    }
+}

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer/HostWorkspace/FileBasedProgramsWorkspaceProviderFactory.cs

@@ -0,0 +1,43 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Composition;
+using Microsoft.CodeAnalysis.Host;
+using Microsoft.CodeAnalysis.Host.Mef;
+using Microsoft.CodeAnalysis.LanguageServer.Handler;
+using Microsoft.CodeAnalysis.LanguageServer.HostWorkspace.ProjectTelemetry;
+using Microsoft.CodeAnalysis.MetadataAsSource;
+using Microsoft.CodeAnalysis.Options;
+using Microsoft.CodeAnalysis.ProjectSystem;
+using Microsoft.CodeAnalysis.Shared.TestHooks;
+using Microsoft.CommonLanguageServerProtocol.Framework;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.CodeAnalysis.LanguageServer.HostWorkspace;
+
+/// <summary>
+/// Service to create <see cref="LspMiscellaneousFilesWorkspaceProvider"/> instances.
+/// This is not exported as a <see cref="ILspServiceFactory"/> as it requires
+/// special base language server dependencies such as the <see cref="HostServices"/>
+/// </summary>
+[ExportCSharpVisualBasicStatelessLspService(typeof(ILspMiscellaneousFilesWorkspaceProviderFactory), WellKnownLspServerKinds.CSharpVisualBasicLspServer), Shared]
+[method: ImportingConstructor]
+[method: Obsolete(MefConstruction.ImportingConstructorMessage, error: true)]
+internal sealed class FileBasedProgramsWorkspaceProviderFactory(
+    IMetadataAsSourceFileService metadataAsSourceFileService,
+    LanguageServerWorkspaceFactory workspaceFactory,
+    IFileChangeWatcher fileChangeWatcher,
+    IGlobalOptionService globalOptionService,
+    ILoggerFactory loggerFactory,
+    IAsynchronousOperationListenerProvider listenerProvider,
+    ProjectLoadTelemetryReporter projectLoadTelemetry,
+    ServerConfigurationFactory serverConfigurationFactory,
+    BinlogNamer binlogNamer) : ILspMiscellaneousFilesWorkspaceProviderFactory
+{
+    public ILspMiscellaneousFilesWorkspaceProvider CreateLspMiscellaneousFilesWorkspaceProvider(ILspServices lspServices, HostServices hostServices)
+    {
+        // TODO(fbp): check feature flag and maybe give base misc workspace
+        return new FileBasedProgramsProjectSystem(lspServices, metadataAsSourceFileService, workspaceFactory, fileChangeWatcher, globalOptionService, loggerFactory, listenerProvider, projectLoadTelemetry, serverConfigurationFactory, binlogNamer);
+    }
+}