diff --git a/CodeEdit/Features/Documents/CodeFileDocument/CodeFileDocument.swift b/CodeEdit/Features/Documents/CodeFileDocument/CodeFileDocument.swift
index 8fd36004b..2b3fcfd24 100644
--- a/CodeEdit/Features/Documents/CodeFileDocument/CodeFileDocument.swift
+++ b/CodeEdit/Features/Documents/CodeFileDocument/CodeFileDocument.swift
@@ -50,9 +50,6 @@ final class CodeFileDocument: NSDocument, ObservableObject {
/// See ``CodeEditSourceEditor/CombineCoordinator``.
@Published var contentCoordinator: CombineCoordinator = CombineCoordinator()
- /// Set by ``LanguageServer`` when initialized.
- @Published var lspCoordinator: LSPContentCoordinator?
-
/// Used to override detected languages.
@Published var language: CodeLanguage?
@@ -65,6 +62,9 @@ final class CodeFileDocument: NSDocument, ObservableObject {
/// Document-specific overridden line wrap preference.
@Published var wrapLines: Bool?
+ /// Set up by ``LanguageServer``, conforms this type to ``LanguageServerDocument``.
+ @Published var languageServerObjects: LanguageServerDocumentObjects<CodeFileDocument> = .init()
+
/// The type of data this file document contains.
///
/// If its text content is not nil, a `text` UTType is returned.
@@ -83,9 +83,6 @@ final class CodeFileDocument: NSDocument, ObservableObject {
return type
}
- /// A stable string to use when identifying documents with language servers.
- var languageServerURI: String? { fileURL?.absolutePath }
-
/// Specify options for opening the file such as the initial cursor positions.
/// Nulled by ``CodeFileView`` on first load.
var openOptions: OpenOptions?
@@ -208,6 +205,10 @@ final class CodeFileDocument: NSDocument, ObservableObject {
}
}
+ /// Determines the code language of the document.
+ /// Use ``CodeFileDocument/language`` for the default value before using this. That property is used to override
+ /// the file's language.
+ /// - Returns: The detected code language.
func getLanguage() -> CodeLanguage {
guard let url = fileURL else {
return .default
@@ -223,3 +224,13 @@ final class CodeFileDocument: NSDocument, ObservableObject {
fileURL?.findWorkspace()
}
}
+
+// MARK: LanguageServerDocument
+
+extension CodeFileDocument: LanguageServerDocument {
+ /// A stable string to use when identifying documents with language servers.
+ /// Needs to be a valid URI, so always returns with the `file://` prefix to indicate it's a file URI.
+ var languageServerURI: String? {
+ fileURL?.lspURI
+ }
+}
diff --git a/CodeEdit/Features/Editor/Views/CodeFileView.swift b/CodeEdit/Features/Editor/Views/CodeFileView.swift
index eeb8586cf..ae5e167ad 100644
--- a/CodeEdit/Features/Editor/Views/CodeFileView.swift
+++ b/CodeEdit/Features/Editor/Views/CodeFileView.swift
@@ -19,9 +19,13 @@ struct CodeFileView: View {
/// The current cursor positions in the view
@State private var cursorPositions: [CursorPosition] = []
+ @State private var treeSitterClient: TreeSitterClient = TreeSitterClient()
+
/// Any coordinators passed to the view.
private var textViewCoordinators: [TextViewCoordinator]
+ @State private var highlightProviders: [any HighlightProviding] = []
+
@AppSettings(\.textEditing.defaultTabWidth)
var defaultTabWidth
@AppSettings(\.textEditing.indentOption)
@@ -62,9 +66,10 @@ struct CodeFileView: View {
init(codeFile: CodeFileDocument, textViewCoordinators: [TextViewCoordinator] = [], isEditable: Bool = true) {
self._codeFile = .init(wrappedValue: codeFile)
+
self.textViewCoordinators = textViewCoordinators
+ [codeFile.contentCoordinator]
- + [codeFile.lspCoordinator].compactMap({ $0 })
+ + [codeFile.languageServerObjects.textCoordinator].compactMap({ $0 })
self.isEditable = isEditable
if let openOptions = codeFile.openOptions {
@@ -72,6 +77,8 @@ struct CodeFileView: View {
self.cursorPositions = openOptions.cursorPositions
}
+ updateHighlightProviders()
+
codeFile
.contentCoordinator
.textUpdatePublisher
@@ -119,7 +126,7 @@ struct CodeFileView: View {
editorOverscroll: overscroll.overscrollPercentage,
cursorPositions: $cursorPositions,
useThemeBackground: useThemeBackground,
- highlightProviders: [treeSitter],
+ highlightProviders: highlightProviders,
contentInsets: edgeInsets.nsEdgeInsets,
additionalTextInsets: NSEdgeInsets(top: 2, left: 0, bottom: 0, right: 0),
isEditable: isEditable,
@@ -144,6 +151,10 @@ struct CodeFileView: View {
.onChange(of: settingsFont) { newFontSetting in
font = newFontSetting.current
}
+ .onReceive(codeFile.$languageServerObjects) { languageServerObjects in
+ // This will not be called in single-file views (for now) but is safe to listen to either way
+ updateHighlightProviders(lspHighlightProvider: languageServerObjects.highlightProvider)
+ }
}
/// Determines the style of bracket emphasis based on the `bracketEmphasis` setting and the current theme.
@@ -166,6 +177,12 @@ struct CodeFileView: View {
return .underline(color: color)
}
}
+
+ /// Updates the highlight providers array.
+ /// - Parameter lspHighlightProvider: The language server provider, if available.
+ private func updateHighlightProviders(lspHighlightProvider: HighlightProviding? = nil) {
+ highlightProviders = [lspHighlightProvider].compactMap({ $0 }) + [treeSitterClient]
+ }
}
// This extension is kept here because it should not be used elsewhere in the app and may cause confusion
diff --git a/CodeEdit/Features/LSP/Editor/LSPContentCoordinator.swift b/CodeEdit/Features/LSP/Features/DocumentSync/LSPContentCoordinator.swift
similarity index 87%
rename from CodeEdit/Features/LSP/Editor/LSPContentCoordinator.swift
rename to CodeEdit/Features/LSP/Features/DocumentSync/LSPContentCoordinator.swift
index dc17481e6..aed2a7ada 100644
--- a/CodeEdit/Features/LSP/Editor/LSPContentCoordinator.swift
+++ b/CodeEdit/Features/LSP/Features/DocumentSync/LSPContentCoordinator.swift
@@ -19,7 +19,7 @@ import LanguageServerProtocol
/// Language servers expect edits to be sent in chunks (and it helps reduce processing overhead). To do this, this class
/// keeps an async stream around for the duration of its lifetime. The stream is sent edit notifications, which are then
/// chunked into 250ms timed groups before being sent to the ``LanguageServer``.
-class LSPContentCoordinator: TextViewCoordinator, TextViewDelegate {
+class LSPContentCoordinator<DocumentType: LanguageServerDocument>: TextViewCoordinator, TextViewDelegate {
// Required to avoid a large_tuple lint error
private struct SequenceElement: Sendable {
let uri: String
@@ -28,25 +28,27 @@ class LSPContentCoordinator: TextViewCoordinator, TextViewDelegate {
}
private var editedRange: LSPRange?
- private var stream: AsyncStream<SequenceElement>?
private var sequenceContinuation: AsyncStream<SequenceElement>.Continuation?
private var task: Task<Void, Never>?
- weak var languageServer: LanguageServer?
+ weak var languageServer: LanguageServer<DocumentType>?
var documentURI: String
/// Initializes a content coordinator, and begins an async stream of updates
- init(documentURI: String, languageServer: LanguageServer) {
+ init(documentURI: String, languageServer: LanguageServer<DocumentType>) {
self.documentURI = documentURI
self.languageServer = languageServer
- self.stream = AsyncStream { continuation in
- self.sequenceContinuation = continuation
- }
+
+ setUpUpdatesTask()
}
func setUpUpdatesTask() {
task?.cancel()
- guard let stream else { return }
+ // Create this stream here so it's always set up when the text view is set up, rather than only once on init.
+ let stream = AsyncStream { continuation in
+ self.sequenceContinuation = continuation
+ }
+
task = Task.detached { [weak self] in
// Send edit events every 250ms
for await events in stream.chunked(by: .repeating(every: .milliseconds(250), clock: .continuous)) {
diff --git a/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenHighlightProvider.swift b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenHighlightProvider.swift
new file mode 100644
index 000000000..2e391fba4
--- /dev/null
+++ b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenHighlightProvider.swift
@@ -0,0 +1,172 @@
+//
+// SemanticTokenHighlightProvider.swift
+// CodeEdit
+//
+// Created by Khan Winter on 12/26/24.
+//
+
+import Foundation
+import LanguageServerProtocol
+import CodeEditSourceEditor
+import CodeEditTextView
+import CodeEditLanguages
+
+/// Provides semantic token information from a language server for a source editor view.
+///
+/// This class works in tangent with the ``LanguageServer`` class to ensure we don't unnecessarily request new tokens
+/// if the document isn't updated. The ``LanguageServer`` will call the
+/// ``SemanticTokenHighlightProvider/documentDidChange`` method, which in turn refreshes the semantic token storage.
+///
+/// That behavior may not be intuitive due to the
+/// ``SemanticTokenHighlightProvider/applyEdit(textView:range:delta:completion:)`` method. One might expect this class
+/// to respond to that method immediately, but it does not. It instead stores the completion passed in that method until
+/// it can respond to the edit with invalidated indices.
+final class SemanticTokenHighlightProvider<
+ Storage: GenericSemanticTokenStorage,
+ DocumentType: LanguageServerDocument
+>: HighlightProviding {
+ enum HighlightError: Error {
+ case lspRangeFailure
+ }
+
+ typealias EditCallback = @MainActor (Result<IndexSet, any Error>) -> Void
+ typealias HighlightCallback = @MainActor (Result<[HighlightRange], any Error>) -> Void
+
+ private let tokenMap: SemanticTokenMap
+ private let documentURI: String
+ private weak var languageServer: LanguageServer<DocumentType>?
+ private weak var textView: TextView?
+
+ private var lastEditCallback: EditCallback?
+ private var pendingHighlightCallbacks: [HighlightCallback] = []
+ private var storage: Storage
+
+ var documentRange: NSRange {
+ textView?.documentRange ?? .zero
+ }
+
+ init(tokenMap: SemanticTokenMap, languageServer: LanguageServer<DocumentType>, documentURI: String) {
+ self.tokenMap = tokenMap
+ self.languageServer = languageServer
+ self.documentURI = documentURI
+ self.storage = Storage()
+ }
+
+ // MARK: - Language Server Content Lifecycle
+
+ /// Called when the language server finishes sending a document update.
+ ///
+ /// This method first checks if this object has any semantic tokens. If not, requests new tokens and responds to the
+ /// `pendingHighlightCallbacks` queue with cancellation errors, causing the highlighter to re-query those indices.
+ ///
+ /// If this object already has some tokens, it determines whether or not we can request a token delta and
+ /// performs the request.
+ func documentDidChange() async throws {
+ guard let languageServer, let textView else {
+ return
+ }
+
+ guard storage.hasReceivedData else {
+ // We have no semantic token info, request it!
+ try await requestTokens(languageServer: languageServer, textView: textView)
+ await MainActor.run {
+ for callback in pendingHighlightCallbacks {
+ callback(.failure(HighlightProvidingError.operationCancelled))
+ }
+ pendingHighlightCallbacks.removeAll()
+ }
+ return
+ }
+
+ // The document was updated. Update our token cache and send the invalidated ranges for the editor to handle.
+ if let lastResultId = storage.lastResultId {
+ try await requestDeltaTokens(languageServer: languageServer, textView: textView, lastResultId: lastResultId)
+ return
+ }
+
+ try await requestTokens(languageServer: languageServer, textView: textView)
+ }
+
+ // MARK: - LSP Token Requests
+
+ /// Requests and applies a token delta. Requires a previous response identifier.
+ private func requestDeltaTokens(
+ languageServer: LanguageServer<DocumentType>,
+ textView: TextView,
+ lastResultId: String
+ ) async throws {
+ guard let response = try await languageServer.requestSemanticTokens(
+ for: documentURI,
+ previousResultId: lastResultId
+ ) else {
+ return
+ }
+ switch response {
+ case let .optionA(tokenData):
+ await applyEntireResponse(tokenData, callback: lastEditCallback)
+ case let .optionB(deltaData):
+ await applyDeltaResponse(deltaData, callback: lastEditCallback, textView: textView)
+ }
+ }
+
+ /// Requests and applies tokens for an entire document. This does not require a previous response id, and should be
+ /// used in place of `requestDeltaTokens` when that's the case.
+ private func requestTokens(languageServer: LanguageServer<DocumentType>, textView: TextView) async throws {
+ guard let response = try await languageServer.requestSemanticTokens(for: documentURI) else {
+ return
+ }
+ await applyEntireResponse(response, callback: lastEditCallback)
+ }
+
+ // MARK: - Apply LSP Response
+
+ /// Applies a delta response from the LSP to our storage.
+ private func applyDeltaResponse(_ data: SemanticTokensDelta, callback: EditCallback?, textView: TextView?) async {
+ let lspRanges = storage.applyDelta(data)
+ lastEditCallback = nil // Don't use this callback again.
+ await MainActor.run {
+ let ranges = lspRanges.compactMap { textView?.nsRangeFrom($0) }
+ callback?(.success(IndexSet(ranges: ranges)))
+ }
+ }
+
+ private func applyEntireResponse(_ data: SemanticTokens, callback: EditCallback?) async {
+ storage.setData(data)
+ lastEditCallback = nil // Don't use this callback again.
+ await callback?(.success(IndexSet(integersIn: documentRange)))
+ }
+
+ // MARK: - Highlight Provider Conformance
+
+ func setUp(textView: TextView, codeLanguage: CodeLanguage) {
+ // Send off a request to get the initial token data
+ self.textView = textView
+ Task {
+ try await self.documentDidChange()
+ }
+ }
+
+ func applyEdit(textView: TextView, range: NSRange, delta: Int, completion: @escaping EditCallback) {
+ if let lastEditCallback {
+ lastEditCallback(.success(IndexSet())) // Don't throw a cancellation error
+ }
+ lastEditCallback = completion
+ }
+
+ func queryHighlightsFor(textView: TextView, range: NSRange, completion: @escaping HighlightCallback) {
+ guard storage.hasReceivedData else {
+ pendingHighlightCallbacks.append(completion)
+ return
+ }
+
+ guard let lspRange = textView.lspRangeFrom(nsRange: range) else {
+ completion(.failure(HighlightError.lspRangeFailure))
+ return
+ }
+ let rawTokens = storage.getTokensFor(range: lspRange)
+ let highlights = tokenMap
+ .decode(tokens: rawTokens, using: textView)
+ .filter({ $0.capture != nil || !$0.modifiers.isEmpty })
+ completion(.success(highlights))
+ }
+}
diff --git a/CodeEdit/Features/LSP/Editor/SemanticTokenMap.swift b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenMap.swift
similarity index 80%
rename from CodeEdit/Features/LSP/Editor/SemanticTokenMap.swift
rename to CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenMap.swift
index 5a196cf60..317068a2d 100644
--- a/CodeEdit/Features/LSP/Editor/SemanticTokenMap.swift
+++ b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenMap.swift
@@ -45,20 +45,31 @@ struct SemanticTokenMap: Sendable { // swiftlint:enable line_length
/// Decodes the compressed semantic token data into a `HighlightRange` type for use in an editor.
/// This is marked main actor to prevent runtime errors, due to the use of the actor-isolated `rangeProvider`.
/// - Parameters:
- /// - tokens: Semantic tokens from a language server.
+ /// - tokens: Encoded semantic tokens type from a language server.
/// - rangeProvider: The provider to use to translate token ranges to text view ranges.
/// - Returns: An array of decoded highlight ranges.
@MainActor
func decode(tokens: SemanticTokens, using rangeProvider: SemanticTokenMapRangeProvider) -> [HighlightRange] {
- tokens.decode().compactMap { token in
+ return decode(tokens: tokens.decode(), using: rangeProvider)
+ }
+
+ /// Decodes the compressed semantic token data into a `HighlightRange` type for use in an editor.
+ /// This is marked main actor to prevent runtime errors, due to the use of the actor-isolated `rangeProvider`.
+ /// - Parameters:
+ /// - tokens: Decoded semantic tokens from a language server.
+ /// - rangeProvider: The provider to use to translate token ranges to text view ranges.
+ /// - Returns: An array of decoded highlight ranges.
+ @MainActor
+ func decode(tokens: [SemanticToken], using rangeProvider: SemanticTokenMapRangeProvider) -> [HighlightRange] {
+ tokens.compactMap { token in
guard let range = rangeProvider.nsRangeFrom(line: token.line, char: token.char, length: token.length) else {
return nil
}
+ // Only modifiers are bit packed, capture types are given as a simple index into the ``tokenTypeMap``
let modifiers = decodeModifier(token.modifiers)
- // Capture types are indicated by the index of the set bit.
- let type = token.type > 0 ? Int(token.type.trailingZeroBitCount) : -1 // Don't try to decode 0
+ let type = Int(token.type)
let capture = tokenTypeMap.indices.contains(type) ? tokenTypeMap[type] : nil
return HighlightRange(
diff --git a/CodeEdit/Features/LSP/Editor/SemanticTokenMapRangeProvider.swift b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenMapRangeProvider.swift
similarity index 100%
rename from CodeEdit/Features/LSP/Editor/SemanticTokenMapRangeProvider.swift
rename to CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenMapRangeProvider.swift
diff --git a/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenStorage/GenericSemanticTokenStorage.swift b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenStorage/GenericSemanticTokenStorage.swift
new file mode 100644
index 000000000..ecfcb3932
--- /dev/null
+++ b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenStorage/GenericSemanticTokenStorage.swift
@@ -0,0 +1,25 @@
+//
+// GenericSemanticTokenStorage.swift
+// CodeEdit
+//
+// Created by Khan Winter on 12/26/24.
+//
+
+import Foundation
+import LanguageServerProtocol
+import CodeEditSourceEditor
+
+/// Defines a protocol for an object to provide storage for semantic tokens.
+///
+/// There is only one concrete type that conforms to this in CE, but this protocol is useful in testing.
+/// See ``SemanticTokenStorage``.
+protocol GenericSemanticTokenStorage: AnyObject {
+ var lastResultId: String? { get }
+ var hasReceivedData: Bool { get }
+
+ init()
+
+ func getTokensFor(range: LSPRange) -> [SemanticToken]
+ func setData(_ data: borrowing SemanticTokens)
+ func applyDelta(_ deltas: SemanticTokensDelta) -> [SemanticTokenRange]
+}
diff --git a/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenStorage/SemanticTokenRange.swift b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenStorage/SemanticTokenRange.swift
new file mode 100644
index 000000000..6a7bfff6d
--- /dev/null
+++ b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenStorage/SemanticTokenRange.swift
@@ -0,0 +1,13 @@
+//
+// SemanticTokenRange.swift
+// CodeEdit
+//
+// Created by Khan Winter on 12/26/24.
+//
+
+/// Represents the range of a semantic token.
+struct SemanticTokenRange {
+ let line: UInt32
+ let char: UInt32
+ let length: UInt32
+}
diff --git a/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenStorage/SemanticTokenStorage.swift b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenStorage/SemanticTokenStorage.swift
new file mode 100644
index 000000000..3faeae250
--- /dev/null
+++ b/CodeEdit/Features/LSP/Features/SemanticTokens/SemanticTokenStorage/SemanticTokenStorage.swift
@@ -0,0 +1,180 @@
+//
+// SemanticTokenStorage.swift
+// CodeEdit
+//
+// Created by Khan Winter on 12/26/24.
+//
+
+import Foundation
+import LanguageServerProtocol
+import CodeEditSourceEditor
+
+/// This class provides storage for semantic token data.
+///
+/// The LSP spec requires that clients keep the original compressed data to apply delta edits. Delta updates may
+/// appear as a delta to a single number in the compressed array. This class maintains the current state of compressed
+/// tokens and their decoded counterparts. It supports applying delta updates from the language server.
+///
+/// See ``SemanticTokenHighlightProvider`` for it's connection to the editor view.
+final class SemanticTokenStorage: GenericSemanticTokenStorage {
+ /// Represents compressed semantic token data received from a language server.
+ struct CurrentState {
+ let resultId: String?
+ let tokenData: [UInt32]
+ let tokens: [SemanticToken]
+ }
+
+ /// The last received result identifier.
+ var lastResultId: String? {
+ state?.resultId
+ }
+
+ /// Indicates if the storage object has received any data.
+ /// Once `setData` has been called, this returns `true`.
+ /// Other operations will fail without any data in the storage object.
+ var hasReceivedData: Bool {
+ state != nil
+ }
+
+ var state: CurrentState?
+
+ /// Create an empty storage object.
+ init() {
+ state = nil
+ }
+
+ // MARK: - Storage Conformance
+
+ /// Finds all tokens in the given range.
+ /// - Parameter range: The range to query.
+ /// - Returns: All tokens found in the range.
+ func getTokensFor(range: LSPRange) -> [SemanticToken] {
+ guard let state = state, !state.tokens.isEmpty else {
+ return []
+ }
+ var tokens: [SemanticToken] = []
+
+ // Perform a binary search
+ guard var idx = findLowerBound(in: range, data: state.tokens[...]) else {
+ return []
+ }
+
+ while idx < state.tokens.count && state.tokens[idx].startPosition < range.end {
+ tokens.append(state.tokens[idx])
+ idx += 1
+ }
+
+ return tokens
+ }
+
+ /// Clear the current state and set a new one.
+ /// - Parameter data: The semantic tokens to set as the current state.
+ func setData(_ data: borrowing SemanticTokens) {
+ state = CurrentState(resultId: data.resultId, tokenData: data.data, tokens: data.decode())
+ }
+
+ /// Apply a delta object from a language server and returns all token ranges that may need re-drawing.
+ ///
+ /// To calculate invalidated ranges:
+ /// - Grabs all semantic tokens that *will* be updated and invalidates their ranges
+ /// - Loops over all inserted tokens and invalidates their ranges
+ /// This may result in duplicated ranges. It's up to the caller to de-duplicate if necessary. See
+ /// ``SemanticTokenStorage/invalidatedRanges(startIdx:length:data:)``.
+ ///
+ /// - Parameter deltas: The deltas to apply.
+ /// - Returns: Ranges invalidated by the applied deltas.
+ func applyDelta(_ deltas: SemanticTokensDelta) -> [SemanticTokenRange] {
+ assert(state != nil, "State should be set before applying any deltas.")
+ guard var tokenData = state?.tokenData else { return [] }
+ var invalidatedSet: [SemanticTokenRange] = []
+
+ // Apply in reverse order (end to start)
+ for edit in deltas.edits.sorted(by: { $0.start > $1.start }) {
+ invalidatedSet.append(
+ contentsOf: invalidatedRanges(startIdx: edit.start, length: edit.deleteCount, data: tokenData[...])
+ )
+
+ // Apply to our copy of the tokens array
+ if edit.deleteCount > 0 {
+ tokenData.replaceSubrange(Int(edit.start)..<Int(edit.start + edit.deleteCount), with: edit.data ?? [])
+ } else {
+ tokenData.insert(contentsOf: edit.data ?? [], at: Int(edit.start))
+ }
+
+ if edit.data != nil {
+ invalidatedSet.append(
+ contentsOf: invalidatedRanges(
+ startIdx: edit.start,
+ length: UInt(edit.data?.count ?? 0),
+ data: tokenData[...]
+ )
+ )
+ }
+ }
+
+ // Re-decode the updated token data and set the updated state
+ let decodedTokens = SemanticTokens(data: tokenData).decode()
+ state = CurrentState(resultId: deltas.resultId, tokenData: tokenData, tokens: decodedTokens)
+ return invalidatedSet
+ }
+
+ // MARK: - Invalidated Indices
+
+ /// Calculate what document ranges are invalidated due to changes in the compressed token data.
+ ///
+ /// This overestimates invalidated ranges by assuming all tokens touched by a change are invalid. All this does is
+ /// find what tokens are being updated by a delta and return them.
+ ///
+ /// - Parameters:
+ /// - startIdx: The start index of the compressed token data an edits start at.
+ /// - length: The length of any edits.
+ /// - data: A reference to the compressed token data.
+ /// - Returns: All token ranges included in the range of the edit.
+ func invalidatedRanges(startIdx: UInt, length: UInt, data: ArraySlice<UInt32>) -> [SemanticTokenRange] {
+ var ranges: [SemanticTokenRange] = []
+ var idx = startIdx - (startIdx % 5)
+ while idx < startIdx + length {
+ ranges.append(
+ SemanticTokenRange(
+ line: data[Int(idx)],
+ char: data[Int(idx + 1)],
+ length: data[Int(idx + 2)]
+ )
+ )
+ idx += 5
+ }
+ return ranges
+ }
+
+ // MARK: - Binary Search
+
+ /// Finds the lowest index of a `SemanticToken` that is entirely within the specified range.
+ /// - Complexity: Runs an **O(log n)** binary search on the data array.
+ /// - Parameters:
+ /// - range: The range to search in, *not* inclusive.
+ /// - data: The tokens to search. Takes an array slice to avoid unnecessary copying. This must be ordered by
+ /// `startPosition`.
+ /// - Returns: The index in the data array of the lowest data element that lies within the given range, or `nil`
+ /// if none are found.
+ func findLowerBound(in range: LSPRange, data: ArraySlice<SemanticToken>) -> Int? {
+ var low = 0
+ var high = data.count
+
+ // Find the first token with startPosition >= range.start.
+ while low < high {
+ let mid = low + (high - low) / 2
+ if data[mid].startPosition < range.start {
+ low = mid + 1
+ } else {
+ high = mid
+ }
+ }
+
+ // Return the item at `low` if it's valid.
+ if low < data.count && data[low].startPosition >= range.start && data[low].endPosition < range.end {
+ return low
+ }
+
+ return nil
+ }
+}
diff --git a/CodeEdit/Features/LSP/LanguageServer/Capabilities/LanguageServer+DocumentSync.swift b/CodeEdit/Features/LSP/LanguageServer/Capabilities/LanguageServer+DocumentSync.swift
index 563604aa7..be69c6647 100644
--- a/CodeEdit/Features/LSP/LanguageServer/Capabilities/LanguageServer+DocumentSync.swift
+++ b/CodeEdit/Features/LSP/LanguageServer/Capabilities/LanguageServer+DocumentSync.swift
@@ -12,7 +12,7 @@ extension LanguageServer {
/// Tells the language server we've opened a document and would like to begin working with it.
/// - Parameter document: The code document to open.
/// - Throws: Throws errors produced by the language server connection.
- func openDocument(_ document: CodeFileDocument) async throws {
+ func openDocument(_ document: DocumentType) async throws {
do {
guard resolveOpenCloseSupport(), let content = await getIsolatedDocumentContent(document) else {
return
@@ -29,7 +29,7 @@ extension LanguageServer {
)
try await lspInstance.textDocumentDidOpen(DidOpenTextDocumentParams(textDocument: textDocument))
- await updateIsolatedDocument(document, coordinator: openFiles.contentCoordinator(for: document))
+ await updateIsolatedDocument(document)
} catch {
logger.warning("addDocument: Error \(error)")
throw error
@@ -41,9 +41,12 @@ extension LanguageServer {
/// - Throws: Throws errors produced by the language server connection.
func closeDocument(_ uri: String) async throws {
do {
- guard resolveOpenCloseSupport() && openFiles.document(for: uri) != nil else { return }
+ guard resolveOpenCloseSupport(), let document = openFiles.document(for: uri) else { return }
logger.debug("Closing document \(uri, privacy: .private)")
+
openFiles.removeDocument(for: uri)
+ await clearIsolatedDocument(document)
+
let params = DidCloseTextDocumentParams(textDocument: TextDocumentIdentifier(uri: uri))
try await lspInstance.textDocumentDidClose(params)
} catch {
@@ -78,10 +81,11 @@ extension LanguageServer {
func documentChanged(uri: String, changes: [DocumentChange]) async throws {
do {
logger.debug("Document updated, \(uri, privacy: .private)")
+ guard let document = openFiles.document(for: uri) else { return }
+
switch resolveDocumentSyncKind() {
case .full:
- guard let document = openFiles.document(for: uri),
- let content = await getIsolatedDocumentContent(document) else {
+ guard let content = await getIsolatedDocumentContent(document) else {
return
}
let changeEvent = TextDocumentContentChangeEvent(range: nil, rangeLength: nil, text: content.string)
@@ -100,6 +104,10 @@ extension LanguageServer {
case .none:
return
}
+
+ // Let the semantic token provider know about the update.
+ // Note for future: If a related LSP object need notifying about document changes, do it here.
+ try await document.languageServerObjects.highlightProvider?.documentDidChange()
} catch {
logger.warning("closeDocument: Error \(error)")
throw error
@@ -110,18 +118,25 @@ extension LanguageServer {
/// Helper function for grabbing a document's content from the main actor.
@MainActor
- private func getIsolatedDocumentContent(_ document: CodeFileDocument) -> DocumentContent? {
+ private func getIsolatedDocumentContent(_ document: DocumentType) -> DocumentContent? {
guard let uri = document.languageServerURI,
- let language = document.getLanguage().lspLanguage,
let content = document.content?.string else {
return nil
}
- return DocumentContent(uri: uri, language: language, string: content)
+ return DocumentContent(uri: uri, language: document.getLanguage().id.rawValue, string: content)
+ }
+
+ @MainActor
+ private func updateIsolatedDocument(_ document: DocumentType) {
+ document.languageServerObjects = LanguageServerDocumentObjects(
+ textCoordinator: openFiles.contentCoordinator(for: document),
+ highlightProvider: openFiles.semanticHighlighter(for: document)
+ )
}
@MainActor
- private func updateIsolatedDocument(_ document: CodeFileDocument, coordinator: LSPContentCoordinator?) {
- document.lspCoordinator = coordinator
+ private func clearIsolatedDocument(_ document: DocumentType) {
+ document.languageServerObjects = LanguageServerDocumentObjects()
}
// swiftlint:disable line_length
@@ -156,7 +171,7 @@ extension LanguageServer {
// Used to avoid a lint error (`large_tuple`) for the return type of `getIsolatedDocumentContent`
fileprivate struct DocumentContent {
let uri: String
- let language: LanguageIdentifier
+ let language: String
let string: String
}
}
diff --git a/CodeEdit/Features/LSP/LanguageServer/Capabilities/LanguageServer+SemanticTokens.swift b/CodeEdit/Features/LSP/LanguageServer/Capabilities/LanguageServer+SemanticTokens.swift
index 02cb29947..b95098d02 100644
--- a/CodeEdit/Features/LSP/LanguageServer/Capabilities/LanguageServer+SemanticTokens.swift
+++ b/CodeEdit/Features/LSP/LanguageServer/Capabilities/LanguageServer+SemanticTokens.swift
@@ -9,12 +9,9 @@ import Foundation
import LanguageServerProtocol
extension LanguageServer {
- /// Setup and test the validity of a rename operation at a given location
func requestSemanticTokens(for documentURI: String) async throws -> SemanticTokensResponse {
do {
- let params = SemanticTokensParams(
- textDocument: TextDocumentIdentifier(uri: documentURI)
- )
+ let params = SemanticTokensParams(textDocument: TextDocumentIdentifier(uri: documentURI))
return try await lspInstance.semanticTokensFull(params)
} catch {
logger.warning("requestSemanticTokens full: Error \(error)")
@@ -22,19 +19,6 @@ extension LanguageServer {
}
}
- func requestSemanticTokens(
- for documentURI: String,
- forRange range: LSPRange
- ) async throws -> SemanticTokensResponse {
- do {
- let params = SemanticTokensRangeParams(textDocument: TextDocumentIdentifier(uri: documentURI), range: range)
- return try await lspInstance.semanticTokensRange(params)
- } catch {
- logger.warning("requestSemanticTokens range: Error \(error)")
- throw error
- }
- }
-
func requestSemanticTokens(
for documentURI: String,
previousResultId: String
diff --git a/CodeEdit/Features/LSP/LanguageServer/LanguageServer.swift b/CodeEdit/Features/LSP/LanguageServer/LanguageServer.swift
index eab8be550..a7c48bb25 100644
--- a/CodeEdit/Features/LSP/LanguageServer/LanguageServer.swift
+++ b/CodeEdit/Features/LSP/LanguageServer/LanguageServer.swift
@@ -11,8 +11,11 @@ import LanguageClient
import LanguageServerProtocol
import OSLog
-class LanguageServer {
- static let logger = Logger(subsystem: Bundle.main.bundleIdentifier ?? "", category: "LanguageServer")
+/// A client for language servers.
+class LanguageServer<DocumentType: LanguageServerDocument> {
+ static var logger: Logger { // types with associated types cannot have constant static properties
+ Logger(subsystem: Bundle.main.bundleIdentifier ?? "", category: "LanguageServer")
+ }
let logger: Logger
/// Identifies which language the server belongs to
@@ -25,7 +28,7 @@ class LanguageServer {
/// Tracks documents and their associated objects.
/// Use this property when adding new objects that need to track file data, or have a state associated with the
/// language server and a document. For example, the content coordinator.
- let openFiles: LanguageServerFileMap
+ let openFiles: LanguageServerFileMap<DocumentType>
/// Maps the language server's highlight config to one CodeEdit can read. See ``SemanticTokenMap``.
let highlightMap: SemanticTokenMap?
@@ -152,13 +155,13 @@ class LanguageServer {
// https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#semanticTokensClientCapabilities
semanticTokens: SemanticTokensClientCapabilities(
dynamicRegistration: false,
- requests: .init(range: true, delta: true),
+ requests: .init(range: false, delta: true),
tokenTypes: SemanticTokenTypes.allStrings,
tokenModifiers: SemanticTokenModifiers.allStrings,
formats: [.relative],
overlappingTokenSupport: true,
multilineTokenSupport: true,
- serverCancelSupport: true,
+ serverCancelSupport: false,
augmentsSyntaxTokens: true
)
)
@@ -218,7 +221,7 @@ class LanguageServer {
processId: nil,
locale: nil,
rootPath: nil,
- rootUri: workspacePath,
+ rootUri: "file://" + workspacePath, // Make it a URI
initializationOptions: [],
capabilities: capabilities,
trace: nil,
diff --git a/CodeEdit/Features/LSP/LanguageServer/LanguageServerFileMap.swift b/CodeEdit/Features/LSP/LanguageServer/LanguageServerFileMap.swift
index c681e894a..fd71a06b7 100644
--- a/CodeEdit/Features/LSP/LanguageServer/LanguageServerFileMap.swift
+++ b/CodeEdit/Features/LSP/LanguageServer/LanguageServerFileMap.swift
@@ -9,39 +9,55 @@ import Foundation
import LanguageServerProtocol
/// Tracks data associated with files and language servers.
-class LanguageServerFileMap {
+class LanguageServerFileMap<DocumentType: LanguageServerDocument> {
+ typealias HighlightProviderType = SemanticTokenHighlightProvider<SemanticTokenStorage, DocumentType>
+
/// Extend this struct as more objects are associated with a code document.
private struct DocumentObject {
let uri: String
var documentVersion: Int
- var contentCoordinator: LSPContentCoordinator
+ var contentCoordinator: LSPContentCoordinator<DocumentType>
+ var semanticHighlighter: HighlightProviderType?
}
- private var trackedDocuments: NSMapTable<NSString, CodeFileDocument>
+ private var trackedDocuments: NSMapTable<NSString, DocumentType>
private var trackedDocumentData: [String: DocumentObject] = [:]
init() {
- trackedDocuments = NSMapTable<NSString, CodeFileDocument>(valueOptions: [.weakMemory])
+ trackedDocuments = NSMapTable<NSString, DocumentType>(valueOptions: [.weakMemory])
}
// MARK: - Track & Remove Documents
- func addDocument(_ document: CodeFileDocument, for server: LanguageServer) {
+ func addDocument(_ document: DocumentType, for server: LanguageServer<DocumentType>) {
guard let uri = document.languageServerURI else { return }
trackedDocuments.setObject(document, forKey: uri as NSString)
- trackedDocumentData[uri] = DocumentObject(
+ var docData = DocumentObject(
uri: uri,
documentVersion: 0,
- contentCoordinator: LSPContentCoordinator(documentURI: uri, languageServer: server)
+ contentCoordinator: LSPContentCoordinator(
+ documentURI: uri,
+ languageServer: server
+ ),
+ semanticHighlighter: nil
)
+
+ if let tokenMap = server.highlightMap {
+ docData.semanticHighlighter = HighlightProviderType(
+ tokenMap: tokenMap,
+ languageServer: server,
+ documentURI: uri
+ )
+ }
+
+ trackedDocumentData[uri] = docData
}
- func document(for uri: DocumentUri) -> CodeFileDocument? {
- let url = URL(filePath: uri)
- return trackedDocuments.object(forKey: url.absolutePath as NSString)
+ func document(for uri: DocumentUri) -> DocumentType? {
+ return trackedDocuments.object(forKey: uri as NSString)
}
- func removeDocument(for document: CodeFileDocument) {
+ func removeDocument(for document: DocumentType) {
guard let uri = document.languageServerURI else { return }
removeDocument(for: uri)
}
@@ -53,7 +69,7 @@ class LanguageServerFileMap {
// MARK: - Version Number Tracking
- func incrementVersion(for document: CodeFileDocument) -> Int {
+ func incrementVersion(for document: DocumentType) -> Int {
guard let uri = document.languageServerURI else { return 0 }
return incrementVersion(for: uri)
}
@@ -63,7 +79,7 @@ class LanguageServerFileMap {
return trackedDocumentData[uri]?.documentVersion ?? 0
}
- func documentVersion(for document: CodeFileDocument) -> Int? {
+ func documentVersion(for document: DocumentType) -> Int? {
guard let uri = document.languageServerURI else { return nil }
return documentVersion(for: uri)
}
@@ -74,12 +90,19 @@ class LanguageServerFileMap {
// MARK: - Content Coordinator
- func contentCoordinator(for document: CodeFileDocument) -> LSPContentCoordinator? {
+ func contentCoordinator(for document: DocumentType) -> LSPContentCoordinator<DocumentType>? {
guard let uri = document.languageServerURI else { return nil }
return contentCoordinator(for: uri)
}
- func contentCoordinator(for uri: DocumentUri) -> LSPContentCoordinator? {
+ func contentCoordinator(for uri: DocumentUri) -> LSPContentCoordinator<DocumentType>? {
trackedDocumentData[uri]?.contentCoordinator
}
+
+ // MARK: - Semantic Highlighter
+
+ func semanticHighlighter(for document: DocumentType) -> HighlightProviderType? {
+ guard let uri = document.languageServerURI else { return nil }
+ return trackedDocumentData[uri]?.semanticHighlighter
+ }
}
diff --git a/CodeEdit/Features/LSP/LanguageServerDocument.swift b/CodeEdit/Features/LSP/LanguageServerDocument.swift
new file mode 100644
index 000000000..8b4b09a47
--- /dev/null
+++ b/CodeEdit/Features/LSP/LanguageServerDocument.swift
@@ -0,0 +1,23 @@
+//
+// LanguageServerDocument.swift
+// CodeEdit
+//
+// Created by Khan Winter on 2/12/25.
+//
+
+import AppKit
+import CodeEditLanguages
+
+/// A set of properties a language server sets when a document is registered.
+struct LanguageServerDocumentObjects<DocumentType: LanguageServerDocument> {
+ var textCoordinator: LSPContentCoordinator<DocumentType>?
+ var highlightProvider: SemanticTokenHighlightProvider<SemanticTokenStorage, DocumentType>?
+}
+
+/// A protocol that allows a language server to register objects on a text document.
+protocol LanguageServerDocument: AnyObject {
+ var content: NSTextStorage? { get }
+ var languageServerURI: String? { get }
+ var languageServerObjects: LanguageServerDocumentObjects<Self> { get set }
+ func getLanguage() -> CodeLanguage
+}
diff --git a/CodeEdit/Features/LSP/Service/LSPService.swift b/CodeEdit/Features/LSP/Service/LSPService.swift
index 5110c6c3e..df74fb139 100644
--- a/CodeEdit/Features/LSP/Service/LSPService.swift
+++ b/CodeEdit/Features/LSP/Service/LSPService.swift
@@ -42,7 +42,7 @@ import CodeEditLanguages
/// do {
/// guard var languageClient = self.languageClient(for: .python) else {
/// print("Failed to get client")
-/// throw ServerManagerError.languageClientNotFound
+/// throw LSPServiceError.languageClientNotFound
/// }
///
/// let testFilePathStr = ""
@@ -54,7 +54,7 @@ import CodeEditLanguages
/// // Completion example
/// let textPosition = Position(line: 32, character: 18) // Lines and characters start at 0
/// let completions = try await languageClient.requestCompletion(
-/// document: testFileURL.absoluteString,
+/// document: testFileURL.lspURI,
/// position: textPosition
/// )
/// switch completions {
@@ -99,6 +99,8 @@ import CodeEditLanguages
/// ```
@MainActor
final class LSPService: ObservableObject {
+ typealias LanguageServerType = LanguageServer<CodeFileDocument>
+
let logger: Logger = Logger(subsystem: Bundle.main.bundleIdentifier ?? "", category: "LSPService")
struct ClientKey: Hashable, Equatable {
@@ -112,7 +114,7 @@ final class LSPService: ObservableObject {
}
/// Holds the active language clients
- var languageClients: [ClientKey: LanguageServer] = [:]
+ var languageClients: [ClientKey: LanguageServerType] = [:]
/// Holds the language server configurations for all the installed language servers
var languageConfigs: [LanguageIdentifier: LanguageServerBinary] = [:]
/// Holds all the event listeners for each active language client
@@ -162,10 +164,16 @@ final class LSPService: ObservableObject {
}
/// Gets the language client for the specified language
- func languageClient(for languageId: LanguageIdentifier, workspacePath: String) -> LanguageServer? {
+ func languageClient(for languageId: LanguageIdentifier, workspacePath: String) -> LanguageServerType? {
return languageClients[ClientKey(languageId, workspacePath)]
}
+ func languageClient(forDocument url: URL) -> LanguageServerType? {
+ languageClients.values.first(where: { $0.openFiles.document(for: url.lspURI) != nil })
+ }
+
+ // MARK: - Start Server
+
/// Given a language and workspace path, will attempt to start the language server
/// - Parameters:
/// - languageId: The ID of the language server to start.
@@ -174,14 +182,14 @@ final class LSPService: ObservableObject {
func startServer(
for languageId: LanguageIdentifier,
workspacePath: String
- ) async throws -> LanguageServer {
+ ) async throws -> LanguageServerType {
guard let serverBinary = languageConfigs[languageId] else {
logger.error("Couldn't find language sever binary for \(languageId.rawValue)")
throw LSPError.binaryNotFound
}
logger.info("Starting \(languageId.rawValue) language server")
- let server = try await LanguageServer.createServer(
+ let server = try await LanguageServerType.createServer(
for: languageId,
with: serverBinary,
workspacePath: workspacePath
@@ -193,6 +201,8 @@ final class LSPService: ObservableObject {
return server
}
+ // MARK: - Document Management
+
/// Notify all relevant language clients that a document was opened.
/// - Note: Must be invoked after the contents of the file are available.
/// - Parameter document: The code document that was opened.
@@ -203,7 +213,7 @@ final class LSPService: ObservableObject {
return
}
Task {
- let languageServer: LanguageServer
+ let languageServer: LanguageServerType
do {
if let server = self.languageClients[ClientKey(lspLanguage, workspacePath)] {
languageServer = server
@@ -228,21 +238,19 @@ final class LSPService: ObservableObject {
/// Notify all relevant language clients that a document was closed.
/// - Parameter url: The url of the document that was closed
func closeDocument(_ url: URL) {
- guard let languageClient = languageClients.first(where: {
- $0.value.openFiles.document(for: url.absolutePath) != nil
- })?.value else {
- return
- }
+ guard let languageClient = languageClient(forDocument: url) else { return }
Task {
do {
- try await languageClient.closeDocument(url.absolutePath)
+ try await languageClient.closeDocument(url.lspURI)
} catch {
// swiftlint:disable:next line_length
- logger.error("Failed to close document: \(url.absolutePath, privacy: .private), language: \(languageClient.languageId.rawValue). Error \(error)")
+ logger.error("Failed to close document: \(url.lspURI, privacy: .private), language: \(languageClient.languageId.rawValue). Error \(error)")
}
}
}
+ // MARK: - Close Workspace
+
/// Close all language clients for a workspace.
///
/// This is intentionally synchronous so we can exit from the workspace document's ``WorkspaceDocument/close()``
@@ -266,6 +274,8 @@ final class LSPService: ObservableObject {
}
}
+ // MARK: - Stop Servers
+
/// Attempts to stop a running language server. Throws an error if the server is not found
/// or if the language server throws an error while trying to shutdown.
/// - Parameters:
@@ -274,7 +284,7 @@ final class LSPService: ObservableObject {
func stopServer(forLanguage languageId: LanguageIdentifier, workspacePath: String) async throws {
guard let server = server(for: languageId, workspacePath: workspacePath) else {
logger.error("Server not found for language \(languageId.rawValue) during stop operation")
- throw ServerManagerError.serverNotFound
+ throw LSPServiceError.serverNotFound
}
do {
try await server.shutdownAndExit()
@@ -309,12 +319,3 @@ final class LSPService: ObservableObject {
eventListeningTasks.removeAll()
}
}
-
-// MARK: - Errors
-
-enum ServerManagerError: Error {
- case serverNotFound
- case serverStartFailed
- case serverStopFailed
- case languageClientNotFound
-}
diff --git a/CodeEdit/Features/LSP/Service/LSPServiceError.swift b/CodeEdit/Features/LSP/Service/LSPServiceError.swift
new file mode 100644
index 000000000..d542e4d75
--- /dev/null
+++ b/CodeEdit/Features/LSP/Service/LSPServiceError.swift
@@ -0,0 +1,13 @@
+//
+// LSPServiceError.swift
+// CodeEdit
+//
+// Created by Khan Winter on 3/24/25.
+//
+
+enum LSPServiceError: Error {
+ case serverNotFound
+ case serverStartFailed
+ case serverStopFailed
+ case languageClientNotFound
+}
diff --git a/CodeEdit/Utils/Extensions/SemanticToken/SemanticToken+Position.swift b/CodeEdit/Utils/Extensions/SemanticToken/SemanticToken+Position.swift
new file mode 100644
index 000000000..0e700938a
--- /dev/null
+++ b/CodeEdit/Utils/Extensions/SemanticToken/SemanticToken+Position.swift
@@ -0,0 +1,18 @@
+//
+// SemanticToken+Position.swift
+// CodeEdit
+//
+// Created by Khan Winter on 12/26/24.
+//
+
+import LanguageServerProtocol
+
+extension SemanticToken {
+ var startPosition: Position {
+ Position(line: Int(line), character: Int(char))
+ }
+
+ var endPosition: Position {
+ Position(line: Int(line), character: Int(char + length))
+ }
+}
diff --git a/CodeEdit/Utils/Extensions/TextView/TextView+SemanticTokenRangeProvider.swift b/CodeEdit/Utils/Extensions/TextView/TextView+SemanticTokenRangeProvider.swift
index f41060423..976f9970f 100644
--- a/CodeEdit/Utils/Extensions/TextView/TextView+SemanticTokenRangeProvider.swift
+++ b/CodeEdit/Utils/Extensions/TextView/TextView+SemanticTokenRangeProvider.swift
@@ -7,8 +7,13 @@
import Foundation
import CodeEditTextView
+import LanguageServerProtocol
extension TextView: SemanticTokenMapRangeProvider {
+ func nsRangeFrom(_ range: SemanticTokenRange) -> NSRange? {
+ nsRangeFrom(line: range.line, char: range.char, length: range.length)
+ }
+
func nsRangeFrom(line: UInt32, char: UInt32, length: UInt32) -> NSRange? {
guard let line = layoutManager.textLineForIndex(Int(line)) else {
return nil
diff --git a/CodeEdit/Utils/Extensions/URL/URL+LSPURI.swift b/CodeEdit/Utils/Extensions/URL/URL+LSPURI.swift
new file mode 100644
index 000000000..f29f9057c
--- /dev/null
+++ b/CodeEdit/Utils/Extensions/URL/URL+LSPURI.swift
@@ -0,0 +1,18 @@
+//
+// URL+LSPURI.swift
+// CodeEdit
+//
+// Created by Khan Winter on 3/24/25.
+//
+
+import Foundation
+
+extension URL {
+ /// A stable string to use when identifying documents with language servers.
+ /// Needs to be a valid URI, so always returns with the `file://` prefix to indicate it's a file URI.
+ ///
+ /// Use this whenever possible when using USLs in LSP processing if not using the ``LanguageServerDocument`` type.
+ var lspURI: String {
+ return "file://" + absolutePath
+ }
+}
diff --git a/CodeEditTests/Features/LSP/LanguageServer+DocumentTests.swift b/CodeEditTests/Features/LSP/LanguageServer+CodeFileDocument.swift
similarity index 80%
rename from CodeEditTests/Features/LSP/LanguageServer+DocumentTests.swift
rename to CodeEditTests/Features/LSP/LanguageServer+CodeFileDocument.swift
index d5bee0c13..236f2a721 100644
--- a/CodeEditTests/Features/LSP/LanguageServer+DocumentTests.swift
+++ b/CodeEditTests/Features/LSP/LanguageServer+CodeFileDocument.swift
@@ -1,5 +1,5 @@
//
-// LanguageServer+DocumentTests.swift
+// LanguageServer+CodeFileDocument.swift
// CodeEditTests
//
// Created by Khan Winter on 9/9/24.
@@ -13,10 +13,16 @@ import LanguageServerProtocol
@testable import CodeEdit
-final class LanguageServerDocumentTests: XCTestCase {
+/// This is an integration test for notifications relating to the ``CodeFileDocument`` class.
+///
+/// For *unit* tests with the language server class, add tests to the `LanguageServer+DocumentObjects` test class as
+/// it's cleaner and makes correct use of the mock document type.
+final class LanguageServerCodeFileDocumentTests: XCTestCase {
// Test opening documents in CodeEdit triggers creating a language server,
// further opened documents don't create new servers
+ typealias LanguageServerType = LanguageServer<CodeFileDocument>
+
var tempTestDir: URL!
override func setUp() {
@@ -44,7 +50,7 @@ final class LanguageServerDocumentTests: XCTestCase {
}
}
- func makeTestServer() async throws -> (connection: BufferingServerConnection, server: LanguageServer) {
+ func makeTestServer() async throws -> (connection: BufferingServerConnection, server: LanguageServerType) {
let bufferingConnection = BufferingServerConnection()
var capabilities = ServerCapabilities()
capabilities.textDocumentSync = .optionA(
@@ -56,12 +62,12 @@ final class LanguageServerDocumentTests: XCTestCase {
save: nil
)
)
- let server = LanguageServer(
+ let server = LanguageServerType(
languageId: .swift,
binary: .init(execPath: "", args: [], env: nil),
lspInstance: InitializingServer(
server: bufferingConnection,
- initializeParamsProvider: LanguageServer.getInitParams(workspacePath: tempTestDir.path())
+ initializeParamsProvider: LanguageServerType.getInitParams(workspacePath: tempTestDir.path())
),
serverCapabilities: capabilities,
rootPath: tempTestDir
@@ -81,7 +87,7 @@ final class LanguageServerDocumentTests: XCTestCase {
}
func openCodeFile(
- for server: LanguageServer,
+ for server: LanguageServerType,
connection: BufferingServerConnection,
file: CEWorkspaceFile,
syncOption: TwoTypeOption<TextDocumentSyncOptions, TextDocumentSyncKind>?
@@ -95,8 +101,11 @@ final class LanguageServerDocumentTests: XCTestCase {
// This is usually sent from the LSPService
try await server.openDocument(codeFile)
- await waitForClientEventCount(
- 3,
+ await waitForClientState(
+ (
+ [.initialize],
+ [.initialized, .textDocumentDidOpen]
+ ),
connection: connection,
description: "Initialized (2) and opened (1) notification count"
)
@@ -108,15 +117,18 @@ final class LanguageServerDocumentTests: XCTestCase {
return codeFile
}
- func waitForClientEventCount(_ count: Int, connection: BufferingServerConnection, description: String) async {
+ func waitForClientState(
+ _ expectedValue: ([ClientRequest.Method], [ClientNotification.Method]),
+ connection: BufferingServerConnection,
+ description: String
+ ) async {
let expectation = expectation(description: description)
await withTaskGroup(of: Void.self) { group in
+ group.addTask { await self.fulfillment(of: [expectation], timeout: 2) }
group.addTask {
- await self.fulfillment(of: [expectation], timeout: 2)
- }
- group.addTask {
- for await events in connection.clientEventSequence where events.0.count + events.1.count == count {
+ for await events in connection.clientEventSequence
+ where events.0.map(\.method) == expectedValue.0 && events.1.map(\.method) == expectedValue.1 {
expectation.fulfill()
return
}
@@ -124,6 +136,8 @@ final class LanguageServerDocumentTests: XCTestCase {
}
}
+ // MARK: - Open Close
+
@MainActor
func testOpenCloseFileNotifications() async throws {
// Set up test server
@@ -153,30 +167,30 @@ final class LanguageServerDocumentTests: XCTestCase {
file.fileDocument = codeFile
CodeEditDocumentController.shared.addDocument(codeFile)
- await waitForClientEventCount(3, connection: connection, description: "Pre-close event count")
+ await waitForClientState(
+ (
+ [.initialize],
+ [.initialized, .textDocumentDidOpen]
+ ),
+ connection: connection,
+ description: "Pre-close event count"
+ )
// This should then trigger a documentDidClose event
codeFile.close()
- await waitForClientEventCount(4, connection: connection, description: "Post-close event count")
-
- XCTAssertEqual(
- connection.clientRequests.map { $0.method },
- [
- ClientRequest.Method.initialize,
- ]
- )
-
- XCTAssertEqual(
- connection.clientNotifications.map { $0.method },
- [
- ClientNotification.Method.initialized,
- ClientNotification.Method.textDocumentDidOpen,
- ClientNotification.Method.textDocumentDidClose
- ]
+ await waitForClientState(
+ (
+ [.initialize],
+ [.initialized, .textDocumentDidOpen, .textDocumentDidClose]
+ ),
+ connection: connection,
+ description: "Post-close event count"
)
}
+ // MARK: - Test Document Edit
+
/// Assert the changed contents received by the buffered connection
func assertExpectedContentChanges(connection: BufferingServerConnection, changes: [String]) {
var foundChangeContents: [String] = []
@@ -184,9 +198,7 @@ final class LanguageServerDocumentTests: XCTestCase {
for notification in connection.clientNotifications {
switch notification {
case let .textDocumentDidChange(params):
- foundChangeContents.append(contentsOf: params.contentChanges.map { event in
- event.text
- })
+ foundChangeContents.append(contentsOf: params.contentChanges.map(\.text))
default:
continue
}
@@ -231,18 +243,17 @@ final class LanguageServerDocumentTests: XCTestCase {
textView.replaceCharacters(in: NSRange(location: 39, length: 0), with: "World")
// Added one notification
- await waitForClientEventCount(4, connection: connection, description: "Edited notification count")
+ await waitForClientState(
+ (
+ [.initialize],
+ [.initialized, .textDocumentDidOpen, .textDocumentDidChange]
+ ),
+ connection: connection,
+ description: "Edited notification count"
+ )
// Make sure our text view is intact
XCTAssertEqual(textView.string, #"func testFunction() -> String { "Hello World" }"#)
- XCTAssertEqual(
- [
- ClientNotification.Method.initialized,
- ClientNotification.Method.textDocumentDidOpen,
- ClientNotification.Method.textDocumentDidChange
- ],
- connection.clientNotifications.map { $0.method }
- )
// Expect only one change due to throttling.
assertExpectedContentChanges(
@@ -289,18 +300,17 @@ final class LanguageServerDocumentTests: XCTestCase {
textView.replaceCharacters(in: NSRange(location: 39, length: 0), with: "World")
// Throttling means we should receive one edited notification + init notification + didOpen + init request
- await waitForClientEventCount(4, connection: connection, description: "Edited notification count")
+ await waitForClientState(
+ (
+ [.initialize],
+ [.initialized, .textDocumentDidOpen, .textDocumentDidChange]
+ ),
+ connection: connection,
+ description: "Edited notification count"
+ )
// Make sure our text view is intact
XCTAssertEqual(textView.string, #"func testFunction() -> String { "Hello World" }"#)
- XCTAssertEqual(
- [
- ClientNotification.Method.initialized,
- ClientNotification.Method.textDocumentDidOpen,
- ClientNotification.Method.textDocumentDidChange
- ],
- connection.clientNotifications.map { $0.method }
- )
// Expect three content changes.
assertExpectedContentChanges(
diff --git a/CodeEditTests/Features/LSP/LanguageServer+DocumentObjects.swift b/CodeEditTests/Features/LSP/LanguageServer+DocumentObjects.swift
new file mode 100644
index 000000000..76b2e8cf3
--- /dev/null
+++ b/CodeEditTests/Features/LSP/LanguageServer+DocumentObjects.swift
@@ -0,0 +1,80 @@
+//
+// LanguageServer+DocumentObjects.swift
+// CodeEditTests
+//
+// Created by Khan Winter on 2/12/25.
+//
+
+import XCTest
+import CodeEditTextView
+import CodeEditSourceEditor
+import CodeEditLanguages
+import LanguageClient
+import LanguageServerProtocol
+
+@testable import CodeEdit
+
+final class LanguageServerDocumentObjectsTests: XCTestCase {
+ final class MockDocumentType: LanguageServerDocument {
+ var content: NSTextStorage?
+ var languageServerURI: String?
+ var languageServerObjects: LanguageServerDocumentObjects<MockDocumentType>
+
+ init() {
+ self.content = NSTextStorage(string: "hello world")
+ self.languageServerURI = "/test/file/path"
+ self.languageServerObjects = .init()
+ }
+
+ func getLanguage() -> CodeLanguage {
+ .swift
+ }
+ }
+
+ typealias LanguageServerType = LanguageServer<MockDocumentType>
+
+ var document: MockDocumentType!
+ var server: LanguageServerType!
+
+ // MARK: - Set Up
+
+ override func setUp() async throws {
+ var capabilities = ServerCapabilities()
+ capabilities.textDocumentSync = .optionA(.init(openClose: true, change: .full))
+ capabilities.semanticTokensProvider = .optionA(.init(legend: .init(tokenTypes: [], tokenModifiers: [])))
+ server = LanguageServerType(
+ languageId: .swift,
+ binary: .init(execPath: "", args: [], env: nil),
+ lspInstance: InitializingServer(
+ server: BufferingServerConnection(),
+ initializeParamsProvider: LanguageServerType.getInitParams(workspacePath: "/")
+ ),
+ serverCapabilities: capabilities,
+ rootPath: URL(fileURLWithPath: "")
+ )
+ _ = try await server.lspInstance.initializeIfNeeded()
+ document = MockDocumentType()
+ }
+
+ // MARK: - Tests
+
+ func testOpenDocumentRegistersObjects() async throws {
+ try await server.openDocument(document)
+ XCTAssertNotNil(document.languageServerObjects.highlightProvider)
+ XCTAssertNotNil(document.languageServerObjects.textCoordinator)
+ XCTAssertNotNil(server.openFiles.document(for: document.languageServerURI ?? ""))
+ }
+
+ func testCloseDocumentClearsObjects() async throws {
+ guard let languageServerURI = document.languageServerURI else {
+ XCTFail("Language server URI missing on a mock object")
+ return
+ }
+ try await server.openDocument(document)
+ XCTAssertNotNil(server.openFiles.document(for: languageServerURI))
+
+ try await server.closeDocument(languageServerURI)
+ XCTAssertNil(document.languageServerObjects.highlightProvider)
+ XCTAssertNil(document.languageServerObjects.textCoordinator)
+ }
+}
diff --git a/CodeEditTests/Features/LSP/SemanticTokenMapTests.swift b/CodeEditTests/Features/LSP/SemanticTokens/SemanticTokenMapTests.swift
similarity index 79%
rename from CodeEditTests/Features/LSP/SemanticTokenMapTests.swift
rename to CodeEditTests/Features/LSP/SemanticTokens/SemanticTokenMapTests.swift
index 4c941de1a..a9ec5c5a3 100644
--- a/CodeEditTests/Features/LSP/SemanticTokenMapTests.swift
+++ b/CodeEditTests/Features/LSP/SemanticTokens/SemanticTokenMapTests.swift
@@ -10,7 +10,7 @@ import CodeEditSourceEditor
import LanguageServerProtocol
@testable import CodeEdit
-final class SemanticTokenMapTestsTests: XCTestCase {
+final class SemanticTokenMapTests: XCTestCase {
// Ignores the line parameter and just returns a range from the char and length for testing
struct MockRangeProvider: SemanticTokenMapRangeProvider {
func nsRangeFrom(line: UInt32, char: UInt32, length: UInt32) -> NSRange? {
@@ -53,10 +53,10 @@ final class SemanticTokenMapTestsTests: XCTestCase {
// Test decode tokens
let tokens = SemanticTokens(tokens: [
- SemanticToken(line: 0, char: 0, length: 1, type: 0, modifiers: 0b11), // First two indices set
+ SemanticToken(line: 0, char: 0, length: 1, type: 1000000, modifiers: 0b11), // First two indices set
SemanticToken(line: 0, char: 1, length: 2, type: 0, modifiers: 0b100100), // 6th and 3rd indices set
- SemanticToken(line: 0, char: 4, length: 1, type: 0b1, modifiers: 0b101),
- SemanticToken(line: 0, char: 5, length: 1, type: 0b100, modifiers: 0b1010),
+ SemanticToken(line: 0, char: 4, length: 1, type: 1, modifiers: 0b101),
+ SemanticToken(line: 0, char: 5, length: 1, type: 4, modifiers: 0b1010),
SemanticToken(line: 0, char: 7, length: 10, type: 0, modifiers: 0)
])
let decoded = map.decode(tokens: tokens, using: mockProvider)
@@ -69,10 +69,10 @@ final class SemanticTokenMapTestsTests: XCTestCase {
XCTAssertEqual(decoded[4].range, NSRange(location: 7, length: 10), "Decoded range")
XCTAssertEqual(decoded[0].capture, nil, "No Decoded Capture")
- XCTAssertEqual(decoded[1].capture, nil, "No Decoded Capture")
- XCTAssertEqual(decoded[2].capture, .include, "Decoded Capture")
- XCTAssertEqual(decoded[3].capture, .keyword, "Decoded Capture")
- XCTAssertEqual(decoded[4].capture, nil, "No Decoded Capture")
+ XCTAssertEqual(decoded[1].capture, .include, "No Decoded Capture")
+ XCTAssertEqual(decoded[2].capture, .constructor, "Decoded Capture")
+ XCTAssertEqual(decoded[3].capture, .comment, "Decoded Capture")
+ XCTAssertEqual(decoded[4].capture, .include, "No Decoded Capture")
XCTAssertEqual(decoded[0].modifiers, [.declaration, .definition], "Decoded Modifiers")
XCTAssertEqual(decoded[1].modifiers, [.readonly, .defaultLibrary], "Decoded Modifiers")
@@ -92,10 +92,10 @@ final class SemanticTokenMapTestsTests: XCTestCase {
// Test decode tokens
let tokens = SemanticTokens(tokens: [
- SemanticToken(line: 0, char: 0, length: 1, type: 0, modifiers: 0b11), // First two indices set
+ SemanticToken(line: 0, char: 0, length: 1, type: 100, modifiers: 0b11), // First two indices set
SemanticToken(line: 0, char: 1, length: 2, type: 0, modifiers: 0b100100), // 6th and 3rd indices set
- SemanticToken(line: 0, char: 4, length: 1, type: 0b1, modifiers: 0b101),
- SemanticToken(line: 0, char: 5, length: 1, type: 0b100, modifiers: 0b1010),
+ SemanticToken(line: 0, char: 4, length: 1, type: 1, modifiers: 0b101),
+ SemanticToken(line: 0, char: 5, length: 1, type: 4, modifiers: 0b1010),
SemanticToken(line: 0, char: 7, length: 10, type: 0, modifiers: 0)
])
let decoded = map.decode(tokens: tokens, using: mockProvider)
@@ -108,10 +108,10 @@ final class SemanticTokenMapTestsTests: XCTestCase {
XCTAssertEqual(decoded[4].range, NSRange(location: 7, length: 10), "Decoded range")
XCTAssertEqual(decoded[0].capture, nil, "No Decoded Capture")
- XCTAssertEqual(decoded[1].capture, nil, "No Decoded Capture")
- XCTAssertEqual(decoded[2].capture, .include, "Decoded Capture")
- XCTAssertEqual(decoded[3].capture, .keyword, "Decoded Capture")
- XCTAssertEqual(decoded[4].capture, nil, "No Decoded Capture")
+ XCTAssertEqual(decoded[1].capture, .include, "No Decoded Capture")
+ XCTAssertEqual(decoded[2].capture, .constructor, "Decoded Capture")
+ XCTAssertEqual(decoded[3].capture, .comment, "Decoded Capture")
+ XCTAssertEqual(decoded[4].capture, .include, "No Decoded Capture")
XCTAssertEqual(decoded[0].modifiers, [.declaration, .definition], "Decoded Modifiers")
XCTAssertEqual(decoded[1].modifiers, [.readonly, .defaultLibrary], "Decoded Modifiers")
diff --git a/CodeEditTests/Features/LSP/SemanticTokens/SemanticTokenStorageTests.swift b/CodeEditTests/Features/LSP/SemanticTokens/SemanticTokenStorageTests.swift
new file mode 100644
index 000000000..f2d0179ca
--- /dev/null
+++ b/CodeEditTests/Features/LSP/SemanticTokens/SemanticTokenStorageTests.swift
@@ -0,0 +1,199 @@
+//
+// SemanticTokenStorageTests.swift
+// CodeEdit
+//
+// Created by Khan Winter on 12/26/24.
+//
+
+import Foundation
+import Testing
+import CodeEditSourceEditor
+import LanguageServerProtocol
+@testable import CodeEdit
+
+// For easier comparison while setting semantic tokens
+extension SemanticToken: @retroactive Equatable {
+ public static func == (lhs: SemanticToken, rhs: SemanticToken) -> Bool {
+ lhs.type == rhs.type
+ && lhs.modifiers == rhs.modifiers
+ && lhs.line == rhs.line
+ && lhs.char == rhs.char
+ && lhs.length == rhs.length
+ }
+}
+
+@Suite
+struct SemanticTokenStorageTests {
+ let storage = SemanticTokenStorage()
+
+ let semanticTokens = [
+ SemanticToken(line: 0, char: 0, length: 10, type: 0, modifiers: 0),
+ SemanticToken(line: 1, char: 2, length: 5, type: 2, modifiers: 3),
+ SemanticToken(line: 3, char: 8, length: 10, type: 1, modifiers: 0)
+ ]
+
+ @Test
+ func initialState() async throws {
+ #expect(storage.state == nil)
+ #expect(storage.hasReceivedData == false)
+ #expect(storage.lastResultId == nil)
+ }
+
+ @Test
+ func setData() async throws {
+ storage.setData(
+ SemanticTokens(
+ resultId: "1234",
+ tokens: semanticTokens
+ )
+ )
+
+ let state = try #require(storage.state)
+ #expect(state.tokens == semanticTokens)
+ #expect(state.resultId == "1234")
+
+ #expect(storage.lastResultId == "1234")
+ #expect(storage.hasReceivedData == true)
+ }
+
+ @Test
+ func overwriteDataRepeatedly() async throws {
+ let dataToApply: [(String?, [SemanticToken])] = [
+ (nil, semanticTokens),
+ ("1", []),
+ ("2", semanticTokens.dropLast()),
+ ("3", semanticTokens)
+ ]
+ for (resultId, tokens) in dataToApply {
+ storage.setData(SemanticTokens(resultId: resultId, tokens: tokens))
+ let state = try #require(storage.state)
+ #expect(state.tokens == tokens)
+ #expect(state.resultId == resultId)
+ #expect(storage.lastResultId == resultId)
+ #expect(storage.hasReceivedData == true)
+ }
+ }
+
+ @Suite("ApplyDeltas")
+ struct TokensDeltasTests {
+ struct DeltaEdit {
+ let start: Int
+ let deleteCount: Int
+ let data: [Int]
+
+ func makeString() -> String {
+ let dataString = data.map { String($0) }.joined(separator: ",")
+ return "{\"start\": \(start), \"deleteCount\": \(deleteCount), \"data\": [\(dataString)] }"
+ }
+ }
+
+ func makeDelta(resultId: String, edits: [DeltaEdit]) throws -> SemanticTokensDelta {
+ // This is unfortunate, but there's no public initializer for these structs.
+ // So we have to decode them from JSON strings
+ let editsString = edits.map { $0.makeString() }.joined(separator: ",")
+ let deltasJSON = "{ \"resultId\": \"\(resultId)\", \"edits\": [\(editsString)] }"
+ let decoder = JSONDecoder()
+ let deltas = try decoder.decode(SemanticTokensDelta.self, from: Data(deltasJSON.utf8))
+ return deltas
+ }
+
+ let storage: SemanticTokenStorage
+
+ let semanticTokens = [
+ SemanticToken(line: 0, char: 0, length: 10, type: 0, modifiers: 0),
+ SemanticToken(line: 1, char: 2, length: 5, type: 2, modifiers: 3),
+ SemanticToken(line: 3, char: 8, length: 10, type: 1, modifiers: 0)
+ ]
+
+ init() {
+ storage = SemanticTokenStorage()
+ storage.setData(SemanticTokens(tokens: semanticTokens))
+ #expect(storage.state?.tokens == semanticTokens)
+ }
+
+ @Test
+ func applyEmptyDeltasNoChange() throws {
+ let deltas = try makeDelta(resultId: "1", edits: [])
+
+ _ = storage.applyDelta(deltas)
+
+ let state = try #require(storage.state)
+ #expect(state.tokens.count == 3)
+ #expect(state.resultId == "1")
+ #expect(state.tokens == semanticTokens)
+ }
+
+ @Test
+ func applyInsertDeltas() throws {
+ let deltas = try makeDelta(resultId: "1", edits: [.init(start: 0, deleteCount: 0, data: [0, 2, 3, 0, 1])])
+
+ _ = storage.applyDelta(deltas)
+
+ let state = try #require(storage.state)
+ #expect(state.tokens.count == 4)
+ #expect(storage.lastResultId == "1")
+
+ // Should have inserted one at the beginning
+ #expect(state.tokens[0].line == 0)
+ #expect(state.tokens[0].char == 2)
+ #expect(state.tokens[0].length == 3)
+ #expect(state.tokens[0].modifiers == 1)
+
+ // We inserted a delta into the space before this one (at char 2) so this one starts at the same spot
+ #expect(state.tokens[1] == SemanticToken(line: 0, char: 2, length: 10, type: 0, modifiers: 0))
+ #expect(state.tokens[2] == semanticTokens[1])
+ #expect(state.tokens[3] == semanticTokens[2])
+ }
+
+ @Test
+ func applyDeleteOneDeltas() throws {
+ // Delete the second token (semanticTokens[1]) from the initial state.
+ // Each token is represented by 5 numbers, so token[1] starts at raw data index 5.
+ let deltas = try makeDelta(resultId: "2", edits: [.init(start: 5, deleteCount: 5, data: [])])
+ _ = storage.applyDelta(deltas)
+
+ let state = try #require(storage.state)
+ #expect(state.tokens.count == 2)
+ #expect(state.resultId == "2")
+ // The remaining tokens should be the first and third tokens, except we deleted one line between them
+ // so the third token's line is less one
+ #expect(state.tokens[0] == semanticTokens[0])
+ #expect(state.tokens[1] == SemanticToken(line: 2, char: 8, length: 10, type: 1, modifiers: 0))
+ }
+
+ @Test
+ func applyDeleteManyDeltas() throws {
+ // Delete the first two tokens from the initial state.
+ // Token[0] and token[1] together use 10 integers.
+ let deltas = try makeDelta(resultId: "3", edits: [.init(start: 0, deleteCount: 10, data: [])])
+ _ = storage.applyDelta(deltas)
+
+ let state = try #require(storage.state)
+ #expect(state.tokens.count == 1)
+ #expect(state.resultId == "3")
+ // The only remaining token should be the original third token.
+ #expect(state.tokens[0] == SemanticToken(line: 2, char: 8, length: 10, type: 1, modifiers: 0))
+ }
+
+ @Test
+ func applyInsertAndDeleteDeltas() throws {
+ // Combined test: insert a token at the beginning and delete the last token.
+ // Edit 1: Insert a new token at the beginning.
+ let insertion = DeltaEdit(start: 0, deleteCount: 0, data: [0, 2, 3, 0, 1])
+ // Edit 2: Delete the token that starts at raw data index 10 (the third token in the original state).
+ let deletion = DeltaEdit(start: 10, deleteCount: 5, data: [])
+ let deltas = try makeDelta(resultId: "4", edits: [insertion, deletion])
+ _ = storage.applyDelta(deltas)
+
+ let state = try #require(storage.state)
+ #expect(state.tokens.count == 3)
+ #expect(storage.lastResultId == "4")
+ // The new inserted token becomes the first token.
+ #expect(state.tokens[0] == SemanticToken(line: 0, char: 2, length: 3, type: 0, modifiers: 1))
+ // The original first token is shifted (its character offset increased by 2).
+ #expect(state.tokens[1] == SemanticToken(line: 0, char: 2, length: 10, type: 0, modifiers: 0))
+ // The second token from the original state remains unchanged.
+ #expect(state.tokens[2] == semanticTokens[1])
+ }
+ }
+}