diff --git a/src/LanguageServer/Protocol/Handler/SemanticTokens/SemanticTokensHelpers.cs b/src/LanguageServer/Protocol/Handler/SemanticTokens/SemanticTokensHelpers.cs index 48bbb254982d2..cdc848e2f1521 100644 --- a/src/LanguageServer/Protocol/Handler/SemanticTokens/SemanticTokensHelpers.cs +++ b/src/LanguageServer/Protocol/Handler/SemanticTokens/SemanticTokensHelpers.cs @@ -97,13 +97,15 @@ public static async Task ComputeSemanticTokensDataAsync( await GetClassifiedSpansForDocumentAsync( classifiedSpans, document, textSpans, options, cancellationToken).ConfigureAwait(false); - // Classified spans are not guaranteed to be returned in a certain order so we sort them to be safe. - classifiedSpans.Sort(ClassifiedSpanComparer.Instance); - // Multi-line tokens are not supported by VS (tracked by https://devdiv.visualstudio.com/DevDiv/_workitems/edit/1265495). // Roslyn's classifier however can return multi-line classified spans, so we must break these up into single-line spans. ConvertMultiLineToSingleLineSpans(text, classifiedSpans, updatedClassifiedSpans); + // Classified spans are not guaranteed to be returned in a certain order and + // converting multi-line spans to single line spans can change put spans in the wrong order. + // Sort them before we compute the tokens to ensure we return them to the client in the correct order. + updatedClassifiedSpans.Sort(ClassifiedSpanComparer.Instance); + // TO-DO: We should implement support for streaming if LSP adds support for it: // https://devdiv.visualstudio.com/DevDiv/_workitems/edit/1276300 return ComputeTokens(text.Lines, updatedClassifiedSpans, supportsVisualStudioExtensions, tokenTypesToIndex); @@ -146,7 +148,7 @@ private static void ConvertMultiLineToSingleLineSpans(SourceText text, Segmented if (startLine != endLine) { ConvertToSingleLineSpan( - text, classifiedSpans, updatedClassifiedSpans, ref spanIndex, span.ClassificationType, + text, updatedClassifiedSpans, span.ClassificationType, startLine, startOffset, endLine, endOffSet); } else @@ -158,9 +160,7 @@ private static void ConvertMultiLineToSingleLineSpans(SourceText text, Segmented static void ConvertToSingleLineSpan( SourceText text, - SegmentedList originalClassifiedSpans, SegmentedList updatedClassifiedSpans, - ref int spanIndex, string classificationType, int startLine, int startOffset, @@ -202,19 +202,6 @@ static void ConvertToSingleLineSpan( var updatedClassifiedSpan = new ClassifiedSpan(textSpan, classificationType); updatedClassifiedSpans.Add(updatedClassifiedSpan); } - - // Since spans are expected to be ordered, when breaking up a multi-line span, we may have to insert - // other spans in-between. For example, we may encounter this case when breaking up a multi-line verbatim - // string literal containing escape characters: - // var x = @"one "" - // two"; - // The check below ensures we correctly return the spans in the correct order, i.e. 'one', '""', 'two'. - while (spanIndex + 1 < originalClassifiedSpans.Count && - textSpan.Contains(originalClassifiedSpans[spanIndex + 1].TextSpan)) - { - updatedClassifiedSpans.Add(originalClassifiedSpans[spanIndex + 1]); - spanIndex++; - } } } } diff --git a/src/LanguageServer/ProtocolUnitTests/SemanticTokens/AbstractSemanticTokensTests.cs b/src/LanguageServer/ProtocolUnitTests/SemanticTokens/AbstractSemanticTokensTests.cs index 93add1510ac89..b712f2cd50e83 100644 --- a/src/LanguageServer/ProtocolUnitTests/SemanticTokens/AbstractSemanticTokensTests.cs +++ b/src/LanguageServer/ProtocolUnitTests/SemanticTokens/AbstractSemanticTokensTests.cs @@ -35,10 +35,10 @@ private protected static IReadOnlyDictionary GetTokenTypeToIndex(Te return result; } - private protected static async Task RunGetSemanticTokensRangeAsync(TestLspServer testLspServer, LSP.Location caret, LSP.Range range) + private protected static async Task RunGetSemanticTokensRangeAsync(TestLspServer testLspServer, LSP.Location location) { var result = await testLspServer.ExecuteRequestAsync(LSP.Methods.TextDocumentSemanticTokensRangeName, - CreateSemanticTokensRangeParams(caret, range), CancellationToken.None); + CreateSemanticTokensRangeParams(location), CancellationToken.None); Contract.ThrowIfNull(result); return result; } @@ -57,11 +57,11 @@ private static LSP.SemanticTokensFullParams CreateSemanticTokensFullParams(LSP.L TextDocument = new LSP.TextDocumentIdentifier { DocumentUri = caret.DocumentUri } }; - private static LSP.SemanticTokensRangeParams CreateSemanticTokensRangeParams(LSP.Location caret, LSP.Range range) + private static LSP.SemanticTokensRangeParams CreateSemanticTokensRangeParams(LSP.Location location) => new LSP.SemanticTokensRangeParams { - TextDocument = new LSP.TextDocumentIdentifier { DocumentUri = caret.DocumentUri }, - Range = range + TextDocument = new LSP.TextDocumentIdentifier { DocumentUri = location.DocumentUri }, + Range = location.Range }; private static SemanticTokensRangesParams CreateSemanticTokensRangesParams(LSP.Location caret, Range[] ranges) diff --git a/src/LanguageServer/ProtocolUnitTests/SemanticTokens/SemanticTokensRangeTests.cs b/src/LanguageServer/ProtocolUnitTests/SemanticTokens/SemanticTokensRangeTests.cs index 029e55d893df5..35637b0e0b66c 100644 --- a/src/LanguageServer/ProtocolUnitTests/SemanticTokens/SemanticTokensRangeTests.cs +++ b/src/LanguageServer/ProtocolUnitTests/SemanticTokens/SemanticTokensRangeTests.cs @@ -27,15 +27,14 @@ public async Task TestGetSemanticTokensRange_FullDocAsync(bool mutatingLspWorksp { var markup = """ - {|caret:|}// Comment - static class C { } + {|range:// Comment + static class C { }|} """; await using var testLspServer = await CreateTestLspServerAsync( markup, mutatingLspWorkspace, GetCapabilities(isVS)); - var range = new LSP.Range { Start = new Position(0, 0), End = new Position(2, 0) }; - var results = await RunGetSemanticTokensRangeAsync(testLspServer, testLspServer.GetLocations("caret").First(), range); + var results = await RunGetSemanticTokensRangeAsync(testLspServer, testLspServer.GetLocations("range").First()); var expectedResults = new LSP.SemanticTokens(); var tokenTypeToIndex = GetTokenTypeToIndex(testLspServer); @@ -605,4 +604,89 @@ public void TestGetSemanticTokensRange_AssertCustomTokenTypes(bool isVS) Assert.True(schema.AllTokenTypes.Contains(tokenName)); } } + + [Theory, CombinatorialData] + [WorkItem("https://github.com/dotnet/roslyn/issues/74809")] + public async Task TestGetSemanticTokensRange_EmbeddedClassificationVerbatimString(bool mutatingLspWorkspace, bool isVS) + { + var markup = + """ + public class C + { + public void M() + { + {|range:// lang=c#-test + const string Code = @" + class C { + // + }";|} + } + } + """; + await using var testLspServer = await CreateTestLspServerAsync( + markup, mutatingLspWorkspace, GetCapabilities(isVS)); + + var location = testLspServer.GetLocations("range").Single(); + var results = await RunGetSemanticTokensRangeAsync(testLspServer, location); + + var expectedResults = new LSP.SemanticTokens(); + var tokenTypeToIndex = GetTokenTypeToIndex(testLspServer); + if (isVS) + { + expectedResults.Data = + [ + // Line | Char | Len | Token type | Modifier + 4, 8, 15, tokenTypeToIndex[SemanticTokenTypes.Comment], 0, + 1, 8, 5, tokenTypeToIndex[SemanticTokenTypes.Keyword], 0, + 0, 6, 6, tokenTypeToIndex[SemanticTokenTypes.Keyword], 0, + 0, 7, 4, tokenTypeToIndex[ClassificationTypeNames.ConstantName], 1, + 0, 5, 1, tokenTypeToIndex[SemanticTokenTypes.Operator], 0, + 0, 2, 2, tokenTypeToIndex[ClassificationTypeNames.VerbatimStringLiteral],0, + 1, 0, 17, tokenTypeToIndex[SemanticTokenTypes.Namespace], 0, + 0, 8, 5, tokenTypeToIndex[SemanticTokenTypes.Keyword], 0, + 0, 5, 1, tokenTypeToIndex[ClassificationTypeNames.VerbatimStringLiteral],0, + 0, 1, 1, tokenTypeToIndex[ClassificationTypeNames.ClassName], 0, + 0, 1, 1, tokenTypeToIndex[ClassificationTypeNames.VerbatimStringLiteral],0, + 0, 1, 1, tokenTypeToIndex[ClassificationTypeNames.Punctuation], 0, + 1, 0, 12, tokenTypeToIndex[ClassificationTypeNames.VerbatimStringLiteral],0, + 0, 0, 14, tokenTypeToIndex[SemanticTokenTypes.Namespace], 0, + 0, 12, 2, tokenTypeToIndex[SemanticTokenTypes.Comment], 0, + 1, 0, 8, tokenTypeToIndex[ClassificationTypeNames.VerbatimStringLiteral],0, + 0, 0, 9, tokenTypeToIndex[SemanticTokenTypes.Namespace], 0, + 0, 8, 1, tokenTypeToIndex[ClassificationTypeNames.Punctuation], 0, + 0, 1, 1, tokenTypeToIndex[ClassificationTypeNames.VerbatimStringLiteral],0, + 0, 1, 1, tokenTypeToIndex[ClassificationTypeNames.Punctuation], 0, + ]; + } + else + { + expectedResults.Data = + [ + // Line | Char | Len | Token type | Modifier + 4, 8, 15, tokenTypeToIndex[SemanticTokenTypes.Comment], 0, + 1, 8, 5, tokenTypeToIndex[SemanticTokenTypes.Keyword], 0, + 0, 6, 6, tokenTypeToIndex[SemanticTokenTypes.Keyword], 0, + 0, 7, 4, tokenTypeToIndex[CustomLspSemanticTokenNames.ConstantName], 1, + 0, 5, 1, tokenTypeToIndex[SemanticTokenTypes.Operator], 0, + 0, 2, 2, tokenTypeToIndex[CustomLspSemanticTokenNames.StringVerbatim],0, + 1, 0, 17, tokenTypeToIndex[SemanticTokenTypes.Namespace], 0, + 0, 8, 5, tokenTypeToIndex[SemanticTokenTypes.Keyword], 0, + 0, 5, 1, tokenTypeToIndex[CustomLspSemanticTokenNames.StringVerbatim],0, + 0, 1, 1, tokenTypeToIndex[SemanticTokenTypes.Class], 0, + 0, 1, 1, tokenTypeToIndex[CustomLspSemanticTokenNames.StringVerbatim],0, + 0, 1, 1, tokenTypeToIndex[ClassificationTypeNames.Punctuation], 0, + 1, 0, 12, tokenTypeToIndex[CustomLspSemanticTokenNames.StringVerbatim],0, + 0, 0, 14, tokenTypeToIndex[SemanticTokenTypes.Namespace], 0, + 0, 12, 2, tokenTypeToIndex[SemanticTokenTypes.Comment], 0, + 1, 0, 8, tokenTypeToIndex[CustomLspSemanticTokenNames.StringVerbatim],0, + 0, 0, 9, tokenTypeToIndex[SemanticTokenTypes.Namespace], 0, + 0, 8, 1, tokenTypeToIndex[ClassificationTypeNames.Punctuation], 0, + 0, 1, 1, tokenTypeToIndex[CustomLspSemanticTokenNames.StringVerbatim],0, + 0, 1, 1, tokenTypeToIndex[ClassificationTypeNames.Punctuation], 0, + ]; + } + + await VerifyBasicInvariantsAndNoMultiLineTokens(testLspServer, results.Data).ConfigureAwait(false); + AssertEx.Equal(ConvertToReadableFormat(testLspServer.ClientCapabilities, expectedResults.Data), ConvertToReadableFormat(testLspServer.ClientCapabilities, results.Data)); + } }