diff --git a/CodeEdit.xcodeproj/project.pbxproj b/CodeEdit.xcodeproj/project.pbxproj index b983bf921..c36d455bd 100644 --- a/CodeEdit.xcodeproj/project.pbxproj +++ b/CodeEdit.xcodeproj/project.pbxproj @@ -268,6 +268,10 @@ 611192062B08CCF600D4459B /* SearchIndexer+Add.swift in Sources */ = {isa = PBXBuildFile; fileRef = 611192052B08CCF600D4459B /* SearchIndexer+Add.swift */; }; 611192082B08CCFD00D4459B /* SearchIndexer+Terms.swift in Sources */ = {isa = PBXBuildFile; fileRef = 611192072B08CCFD00D4459B /* SearchIndexer+Terms.swift */; }; 6111920C2B08CD0B00D4459B /* SearchIndexer+InternalMethods.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6111920B2B08CD0B00D4459B /* SearchIndexer+InternalMethods.swift */; }; + 6130535C2B23933D00D767E3 /* MemoryIndexingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6130535B2B23933D00D767E3 /* MemoryIndexingTests.swift */; }; + 6130535F2B23A31300D767E3 /* MemorySearchTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6130535E2B23A31300D767E3 /* MemorySearchTests.swift */; }; + 613053652B23A49300D767E3 /* TemporaryFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 613053642B23A49300D767E3 /* TemporaryFile.swift */; }; + 6130536B2B24722C00D767E3 /* AsyncIndexingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6130536A2B24722C00D767E3 /* AsyncIndexingTests.swift */; }; 613DF55E2B08DD5D00E9D902 /* FileHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 613DF55D2B08DD5D00E9D902 /* FileHelper.swift */; }; 61538B902B111FE800A88846 /* String+AppearancesOfSubstring.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61538B8F2B111FE800A88846 /* String+AppearancesOfSubstring.swift */; }; 61538B932B11201900A88846 /* String+Character.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61538B922B11201900A88846 /* String+Character.swift */; }; @@ -768,6 +772,10 @@ 611192052B08CCF600D4459B /* SearchIndexer+Add.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SearchIndexer+Add.swift"; sourceTree = ""; }; 611192072B08CCFD00D4459B /* SearchIndexer+Terms.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SearchIndexer+Terms.swift"; sourceTree = ""; }; 6111920B2B08CD0B00D4459B /* SearchIndexer+InternalMethods.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SearchIndexer+InternalMethods.swift"; sourceTree = ""; }; + 6130535B2B23933D00D767E3 /* MemoryIndexingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MemoryIndexingTests.swift; sourceTree = ""; }; + 6130535E2B23A31300D767E3 /* MemorySearchTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MemorySearchTests.swift; sourceTree = ""; }; + 613053642B23A49300D767E3 /* TemporaryFile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemporaryFile.swift; sourceTree = ""; }; + 6130536A2B24722C00D767E3 /* AsyncIndexingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AsyncIndexingTests.swift; sourceTree = ""; }; 613DF55D2B08DD5D00E9D902 /* FileHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileHelper.swift; sourceTree = ""; }; 61538B8F2B111FE800A88846 /* String+AppearancesOfSubstring.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "String+AppearancesOfSubstring.swift"; sourceTree = ""; }; 61538B922B11201900A88846 /* String+Character.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "String+Character.swift"; sourceTree = ""; }; @@ -1203,6 +1211,7 @@ children = ( 4EE96ECC296059D200FFBEA8 /* Mocks */, 4EE96ECA2960565E00FFBEA8 /* DocumentsUnitTests.swift */, + 613053582B23916D00D767E3 /* Indexer */, ); path = Documents; sourceTree = ""; @@ -2149,6 +2158,17 @@ path = Indexer; sourceTree = ""; }; + 613053582B23916D00D767E3 /* Indexer */ = { + isa = PBXGroup; + children = ( + 6130535B2B23933D00D767E3 /* MemoryIndexingTests.swift */, + 6130535E2B23A31300D767E3 /* MemorySearchTests.swift */, + 6130536A2B24722C00D767E3 /* AsyncIndexingTests.swift */, + 613053642B23A49300D767E3 /* TemporaryFile.swift */, + ); + path = Indexer; + sourceTree = ""; + }; 6C092EDC2A53A63E00489202 /* Views */ = { isa = PBXGroup; children = ( @@ -3371,11 +3391,15 @@ buildActionMask = 2147483647; files = ( 583E528C29361B39001AB554 /* CodeEditUITests.swift in Sources */, + 613053652B23A49300D767E3 /* TemporaryFile.swift in Sources */, 587B60F82934124200D5CD8F /* CEWorkspaceFileManagerTests.swift in Sources */, + 6130535F2B23A31300D767E3 /* MemorySearchTests.swift in Sources */, 587B61012934170A00D5CD8F /* UnitTests_Extensions.swift in Sources */, 283BDCC52972F236002AFF81 /* AcknowledgementsTests.swift in Sources */, 4EE96ECB2960565E00FFBEA8 /* DocumentsUnitTests.swift in Sources */, 4EE96ECE296059E000FFBEA8 /* NSHapticFeedbackPerformerMock.swift in Sources */, + 6130535C2B23933D00D767E3 /* MemoryIndexingTests.swift in Sources */, + 6130536B2B24722C00D767E3 /* AsyncIndexingTests.swift in Sources */, 587B612E293419B700D5CD8F /* CodeFileTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/CodeEdit/Features/Documents/Indexer/SearchIndexer+AsyncController.swift b/CodeEdit/Features/Documents/Indexer/SearchIndexer+AsyncController.swift index 248f6c321..87c1dfda4 100644 --- a/CodeEdit/Features/Documents/Indexer/SearchIndexer+AsyncController.swift +++ b/CodeEdit/Features/Documents/Indexer/SearchIndexer+AsyncController.swift @@ -35,6 +35,32 @@ extension SearchIndexer { } // MARK: - Search + + /// Performs an asynchronous progressive search on the index for the specified query. + /// + /// - Parameters: + /// - query: The search query string. + /// - maxResults: The maximum number of results to retrieve in each chunk. + /// - timeout: The timeout duration for each search operation. Default is 1.0 second. + /// + /// - Returns: An asynchronous stream (`AsyncStream`) of search results in chunks. + /// The search results are returned in the form of a `SearchIndexer.ProgressivSearch.Results` object. + /// + /// This function initiates a progressive search on the index for the specified query + /// and asynchronously yields search results in chunks using an `AsyncStream`. + /// The search continues until there are no more results or the specified timeout is reached. + /// + /// - Warning: Prior to calling this function, + /// ensure that the `index` has been flushed to search within the most up-to-date data. + /// + /// Example usage: + /// ```swift + /// let searchStream = await asyncController.search(query: searchQuery, 20) + /// for try await result in searchStream { + /// // Process each result + /// print(result) + /// } + /// ``` func search( query: String, _ maxResults: Int, @@ -55,6 +81,14 @@ extension SearchIndexer { // MARK: - Add + /// Adds files from an array of TextFile objects to the index asynchronously. + /// + /// - Parameters: + /// - files: An array of TextFile objects containing the information about the files to be added. + /// - flushWhenComplete: A boolean flag indicating whether to flush + /// the index when the operation is complete. Default is `false`. + /// + /// - Returns: An array of booleans indicating the success of adding each file to the index. func addText( files: [TextFile], flushWhenComplete: Bool = false @@ -62,9 +96,11 @@ extension SearchIndexer { var addedFiles = [Bool]() + // Asynchronously iterate through the provided files using a task group await withTaskGroup(of: Bool.self) { taskGroup in for file in files { taskGroup.addTask { + // Add the file to the index and return the success status return self.index.addFileWithText(file.url, text: file.text, canReplace: true) } } @@ -73,12 +109,24 @@ extension SearchIndexer { addedFiles.append(result) } } + if flushWhenComplete { index.flush() } + return addedFiles } + /// Adds files from an array of URLs to the index asynchronously. + /// + /// - Parameters: + /// - urls: An array of URLs representing the file locations to be added to the index. + /// - flushWhenComplete: A boolean flag indicating whether to flush + /// the index when the operation is complete. Default is `false`. + /// + /// - Returns: An array of booleans indicating the success of adding each file to the index. + /// - Warning: Prefer using `addText` when possible as SearchKit does not have the ability + /// to read every file type. For example, it is often not possible to read Swift files. func addFiles( urls: [URL], flushWhenComplete: Bool = false @@ -100,6 +148,16 @@ extension SearchIndexer { return addedURLs } + /// Adds files from a folder specified by the given URL to the index asynchronously. + /// + /// - Parameters: + /// - url: The URL of the folder containing files to be added to the index. + /// - flushWhenComplete: A boolean flag indicating whether to flush + /// the index when the operation is complete. Default is `false`. + /// + /// This function uses asynchronous processing to add files from the specified folder to the index. + /// + /// - Note: Subfolders within the specified folder are also processed. func addFolder( url: URL, flushWhenComplete: Bool = false diff --git a/CodeEditTests/Features/Documents/Indexer/AsyncIndexingTests.swift b/CodeEditTests/Features/Documents/Indexer/AsyncIndexingTests.swift new file mode 100644 index 000000000..ffaa0030c --- /dev/null +++ b/CodeEditTests/Features/Documents/Indexer/AsyncIndexingTests.swift @@ -0,0 +1,76 @@ +// +// AsyncIndexingTests.swift +// CodeEditTests +// +// Created by Tommy Ludwig on 09.12.23. +// + +import XCTest +@testable import CodeEdit + +final class AsyncIndexingTests: XCTestCase { + func testAddDocuments() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let asyncManager = SearchIndexer.AsyncManager(index: index) + let expectation = XCTestExpectation(description: "Async operations completed") + let tempFile1 = TemporaryFile().url + let tempFile2 = TemporaryFile().url + + Task { + let results = await asyncManager.addFiles(urls: [tempFile1, tempFile2]) + XCTAssertEqual(results.count, 2, "Unexpected indexing results.") + asyncManager.index.flush() + let documents = asyncManager.index.documents() + XCTAssertEqual(documents.count, 2) + expectation.fulfill() + } + + wait(for: [expectation], timeout: 2) + } + + func testSearchDocuments() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let asyncManager = SearchIndexer.AsyncManager(index: index) + let expectation = XCTestExpectation(description: "Async operations completed") + + let tempFile1 = SearchIndexer.AsyncManager.TextFile( + url: TemporaryFile().url, + text: "Itaque ratione asperiores." + ) + let tempFile2 = SearchIndexer.AsyncManager.TextFile( + url: TemporaryFile().url, + text: "Perspiciatis perspiciatis rerum ex asperiores." + ) + + Task { + var searchResults = [URL]() + let results = await asyncManager.addText(files: [tempFile1, tempFile2]) + XCTAssertEqual(results.count, 2, "Unexpected indexing results.") + asyncManager.index.flush() + let searchStream = await asyncManager.search(query: "asperiores", 10) + for try await result in searchStream { + let urls: [(URL, Float)] = result.results.compactMap { + ($0.url, $0.score) + } + + for (url, _) in urls { + searchResults.append(url) + } + } + + XCTAssertEqual(searchResults.count, 2) + expectation.fulfill() + } + + wait(for: [expectation], timeout: 2) + } + +} diff --git a/CodeEditTests/Features/Documents/Indexer/MemoryIndexingTests.swift b/CodeEditTests/Features/Documents/Indexer/MemoryIndexingTests.swift new file mode 100644 index 000000000..a823c4ed9 --- /dev/null +++ b/CodeEditTests/Features/Documents/Indexer/MemoryIndexingTests.swift @@ -0,0 +1,147 @@ +// +// MemoryIndexing.swift +// CodeEditTests +// +// Created by Tommy Ludwig on 08.12.23. +// + +import XCTest +@testable import CodeEdit + +final class MemoryIndexingTests: XCTestCase { + func testIndexFile() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let filePath = TemporaryFile().url + + let indexResults = index.addFileWithText(filePath, text: "Hello, World!") + XCTAssert(indexResults) + } + + func testIndexFiles() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let document1 = TemporaryFile().url + let document2 = TemporaryFile().url + + var indexResults = index.addFileWithText(document1, text: "fileContent") + XCTAssert(indexResults) + indexResults = index.addFileWithText(document2, text: "") + XCTAssert(indexResults) + index.flush() + let res = index.cleanUp() + XCTAssertEqual(res, 1) + } + + func testIndexFolder() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let folder = TempFolderManager() + folder.createCustomFolder() + folder.createFiles() + + let indexResults = index.addFolderContent(folderURL: folder.customFolderURL) + XCTAssertEqual(indexResults.count, 2) + + index.flush() + + let searchResults = index.search("file") + XCTAssertEqual(searchResults.count, 2, "Unexpected search results") + } + + func testIndexCleanUp() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let document1 = TemporaryFile().url + let document2 = TemporaryFile().url + + var indexResults = index.addFileWithText(document1, text: "fileContent") + XCTAssert(indexResults) + indexResults = index.addFileWithText(document2, text: "") + XCTAssert(indexResults) + index.flush() + let res = index.cleanUp() + XCTAssertEqual(res, 1) + } + + func testCloseIndex() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let filePath = TemporaryFile().url + + let indexResults = index.addFileWithText(filePath, text: "Hello, World!") + XCTAssert(indexResults) + + index.close() + + let closedIndexResults = index.addFileWithText(filePath, text: "Hello, World") + XCTAssertEqual(closedIndexResults, false) + } + + func testDocumentIsIndex() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let filePath = TemporaryFile().url + + let indexResults = index.addFileWithText(filePath, text: "Hello, World!") + XCTAssert(indexResults) + + let isIndexed = index.documentIndexed(filePath) + XCTAssertEqual(isIndexed, false) + + index.flush() + let isIndexedAfterFlush = index.documentIndexed(filePath) + XCTAssert(isIndexedAfterFlush) + } + + func testSaveAndLoad() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let textFilePath = TemporaryFile().url + + XCTAssertTrue(index.addFileWithText(textFilePath, text: "Illum assumenda iure earum dolorum fugit.")) + + index.flush() + + let searchResults = index.search("earum") + XCTAssertEqual(1, searchResults.count) + XCTAssertEqual(searchResults[0].url, textFilePath) + + // Save the current index. + let savedIndex = index.getAsData() + XCTAssertNotNil(savedIndex, "Failed to save the index.") + + // Close the index, i.e. the index gets deallocated form memory. + index.close() + + // Load the saved index + guard let loadedIndex = SearchIndexer.Memory(data: savedIndex!) else { + XCTFail("Failed to create an index") + return + } + + let savedIndexResult = loadedIndex.search("earum") + XCTAssertEqual(savedIndexResult.count, 1) + } +} diff --git a/CodeEditTests/Features/Documents/Indexer/MemorySearchTests.swift b/CodeEditTests/Features/Documents/Indexer/MemorySearchTests.swift new file mode 100644 index 000000000..459f0b3d6 --- /dev/null +++ b/CodeEditTests/Features/Documents/Indexer/MemorySearchTests.swift @@ -0,0 +1,85 @@ +// +// MemoryIndexSearch.swift +// CodeEditTests +// +// Created by Tommy Ludwig on 08.12.23. +// + +import XCTest +@testable import CodeEdit + +final class MemoryIndexSearchTests: XCTestCase { + func testIndexFileSearch() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let filePath = TemporaryFile().url + + let indexResults = index.addFileWithText(filePath, text: "Hello, World!") + XCTAssert(indexResults) + index.flush() + let progressivSearch = index.progressiveSearch(query: "hello") + let progressivSearchResults = progressivSearch.getNextSearchResultsChunk(limit: 10) + XCTAssertEqual(progressivSearchResults.results.count, 1) + } + + func testIndexFolderSearch() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let folder = TempFolderManager() + folder.createCustomFolder() + folder.createFiles() + + let indexResults = index.addFolderContent(folderURL: folder.customFolderURL) + XCTAssertEqual(indexResults.count, 2) + } + + func testIndexFileWildCardSearch() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let filePath = TemporaryFile().url + + let indexResults = index.addFileWithText(filePath, text: "Hello, World!") + XCTAssert(indexResults) + index.flush() + let progressivSearch = index.progressiveSearch(query: "*ll*") + let progressivSearchResults = progressivSearch.getNextSearchResultsChunk(limit: 10) + XCTAssertEqual(progressivSearchResults.results.count, 1) + } + + func testIndexRemoveDocument() { + guard let index = SearchIndexer.Memory.create() else { + XCTFail("Failed to create an index") + return + } + + let document1 = TemporaryFile().url + let document2 = TemporaryFile().url + XCTAssertTrue(index.addFileWithText(document1, text: "Hello, World!"), "Failed to add docs to index.") + XCTAssertTrue(index.addFileWithText(document2, text: "Hello, Swift!"), "Failed to add docs to index.") + + index.flush() + + let documents = index.documents() + XCTAssertEqual(documents.count, 2) + + let searchResults = index.search("Hello") + XCTAssertEqual(searchResults.count, 2, "Unexpected search results.") + + let removeResult = index.removeDocument(url: document1) + XCTAssertTrue(removeResult, "Failed to remove documents.") + + index.flush() + + let searchResultsAfterFlush = index.search("Hello") + XCTAssertEqual(searchResultsAfterFlush.count, 1, "Unexpected search results.") + } +} diff --git a/CodeEditTests/Features/Documents/Indexer/TemporaryFile.swift b/CodeEditTests/Features/Documents/Indexer/TemporaryFile.swift new file mode 100644 index 000000000..8e9017b3d --- /dev/null +++ b/CodeEditTests/Features/Documents/Indexer/TemporaryFile.swift @@ -0,0 +1,101 @@ +// +// TemporaryFile.swift +// CodeEditTests +// +// Created by Tommy Ludwig on 08.12.23. +// + +import Foundation + +/// A utility class representing a temporary file with automatic cleanup upon deallocation. +/// +/// This class provides a convenient way to create a temporary +/// file with a unique name and automatically removes the file when the `TemporaryFile` instance is deallocated. +/// +/// Example usage: +/// ```swift +/// let tempFile = TemporaryFile() +/// // Use tempFile.url for file operations +/// // The file will be automatically removed when tempFile is no longer in use. +/// ``` +class TemporaryFile { + /// The URL of the temporary file. + let url: URL = { + let folder = NSTemporaryDirectory() + let name = UUID().uuidString + + return NSURL.fileURL(withPathComponents: [folder, name])! as URL + }() + + /// Deinitializes the `TemporaryFile` instance and removes the associated temporary file from the filesystem. + deinit { + try? FileManager.default.removeItem(at: url) + } +} + +/// A utility class for managing a temporary folder with customizable files. +/// +/// The `TempFolderManager` class facilitates the creation, management, +/// and cleanup of a temporary folder with associated files. +/// +/// Example usage: +/// ```swift +/// let folderManager = TempFolderManager() +/// folderManager.createCustomFolder() +/// folderManager.createFiles() +/// // Use files within the custom folder as needed +/// // The folder and files will be automatically cleaned up upon the `TempFolderManager` instance deinitialization. +/// ``` +class TempFolderManager { + let temporaryDirectoryURL: URL + let customFolderURL: URL + + init() { + self.temporaryDirectoryURL = URL(fileURLWithPath: NSTemporaryDirectory()) + self.customFolderURL = temporaryDirectoryURL.appendingPathComponent("TestingFolder") + } + + deinit { + cleanup() + } + + func createCustomFolder() { + do { + try FileManager.default.createDirectory( + at: customFolderURL, + withIntermediateDirectories: true, + attributes: nil + ) + } catch { + print("Error creating directory: \(error)") + } + } + + func createFiles() { + let file1URL = customFolderURL.appendingPathComponent("file1.txt") + let file2URL = customFolderURL.appendingPathComponent("file2.txt") + + let file1Content = "This is file 1" + let file2Content = "This is file 2" + + do { + try file1Content.write(to: file1URL, atomically: true, encoding: .utf8) + try file2Content.write(to: file2URL, atomically: true, encoding: .utf8) + } catch { + print("Error writing to file: \(error)") + } + } + + func cleanup() { + do { + let file1URL = customFolderURL.appendingPathComponent("file1.txt") + let file2URL = customFolderURL.appendingPathComponent("file2.txt") + + try FileManager.default.removeItem(at: file1URL) + try FileManager.default.removeItem(at: file2URL) + try FileManager.default.removeItem(at: customFolderURL) + } catch { + print("Error removing item: \(error)") + } + } +}