Implement multimodal( FTS, vector, hybrid) search capability

electron/main/vector-database/ipcHandlers.ts

@@ -10,7 +10,7 @@ import { StoreSchema } from '../electron-store/storeConfig'
 import { startWatchingDirectory, updateFileListForRenderer } from '../filesystem/filesystem'
 
 import { rerankSearchedEmbeddings } from './embeddings'
-import { DBEntry, DatabaseFields } from './schema'
+import { DBEntry, DatabaseFields, DBQueryResult } from './schema'
 import { RepopulateTableWithMissingItems } from './tableHelperFunctions'
 
 export interface PromptWithRagResults {
@@ -37,6 +37,24 @@ export const registerDBSessionHandlers = (store: Store<StoreSchema>, _windowMana
     return searchResults
   })
 
+  ipcMain.handle(
+    'multi-modal-search',
+    async (
+      event,
+      query: string,
+      limit: number,
+      searchType: 'vector' | 'text' | 'hybrid',
+      filter?: string,
+    ): Promise<{ vectorResults: DBQueryResult[]; textResults: DBQueryResult[] }> => {
+      const windowInfo = windowManager.getWindowInfoForContents(event.sender)
+      if (!windowInfo) {
+        throw new Error('Window info not found.')
+      }
+      const searchResults = await windowInfo.dbTableClient.multiModalSearch(query, limit, searchType, filter)
+      return searchResults
+    },
+  )
+
   ipcMain.handle('index-files-in-directory', async (event) => {
     const windowInfo = windowManager.getWindowInfoForContents(event.sender)
     if (!windowInfo) {

electron/main/vector-database/lanceTableWrapper.ts

@@ -118,6 +118,40 @@ class LanceDBTableWrapper {
     const mapped = rawResults.map(convertRecordToDBType<DBEntry>)
     return mapped as DBEntry[]
   }
+
+  async multiModalSearch(
+    query: string,
+    limit: number,
+    searchType: 'vector' | 'text' | 'hybrid' = 'vector',
+    filter?: string,
+  ): Promise<{ vectorResults: DBQueryResult[]; textResults: DBQueryResult[] }> {
+    let vectorResults: DBQueryResult[] = []
+    let textResults: DBQueryResult[] = []
+
+    if (searchType === 'vector' || searchType === 'hybrid') {
+      const vectorQuery = await this.lanceTable.search(query).metricType(MetricType.Cosine).limit(limit)
+      if (filter) {
+        vectorQuery.prefilter(true).filter(filter)
+      }
+      const rawVectorResults = await vectorQuery.execute()
+      vectorResults = rawVectorResults
+        .map(convertRecordToDBType<DBQueryResult>)
+        .filter((r): r is DBQueryResult => r !== null)
+    }
+
+    if (searchType === 'text' || searchType === 'hybrid') {
+      const sanitizedTextQuery = sanitizePathForDatabase(query)
+      const textFilter = filter
+        ? `${filter} AND content LIKE '%${sanitizedTextQuery}%'`
+        : `content LIKE '%${sanitizedTextQuery}%'`
+      const rawTextResults = await this.lanceTable.filter(textFilter).limit(limit).execute()
+      textResults = rawTextResults
+        .map(convertRecordToDBType<DBQueryResult>)
+        .filter((r): r is DBQueryResult => r !== null)
+    }
+
+    return { vectorResults, textResults }
+  }
 }
 
 export default LanceDBTableWrapper

electron/preload/index.ts

@@ -22,6 +22,15 @@ function createIPCHandler<T extends (...args: any[]) => any>(channel: string): I
 
 const database = {
   search: createIPCHandler<(query: string, limit: number, filter?: string) => Promise<DBQueryResult[]>>('search'),
+  multiModalSearch:
+    createIPCHandler<
+      (
+        query: string,
+        limit: number,
+        searchType: 'vector' | 'text' | 'hybrid',
+        filter?: string,
+      ) => Promise<{ vectorResults: DBQueryResult[]; textResults: DBQueryResult[] }>
+    >('multi-modal-search'),
   deleteLanceDBEntriesByFilePath: createIPCHandler<(filePath: string) => Promise<void>>(
     'delete-lance-db-entries-by-filepath',
   ),

src/components/File/DBResultPreview.tsx

@@ -62,10 +62,15 @@ export const DBSearchPreview: React.FC<DBSearchPreviewProps> = ({ dbResult: entr
         <MarkdownRenderer content={entry.content} />
       </div>
       <div className="mt-2 text-xs text-gray-400">
-        {fileName && <span className="text-xs text-gray-400">{fileName} </span>} | Similarity:{' '}
+        {fileName && <span className="text-xs text-gray-400">{fileName} </span>}
         {/* eslint-disable-next-line no-underscore-dangle */}
-        {cosineDistanceToPercentage(entry._distance)}% |{' '}
-        {modified && <span className="text-xs text-gray-400">Modified {modified}</span>}
+        {entry._distance != null && (
+          <>
+            {/* eslint-disable-next-line no-underscore-dangle */}| Similarity:{' '}
+            {cosineDistanceToPercentage(entry._distance)}%{' '}
+          </>
+        )}{' '}
+        | {modified && <span className="text-xs text-gray-400">Modified {modified}</span>}
       </div>
     </div>
   )

src/components/MainPage.tsx

@@ -52,7 +52,7 @@ const MainPageContent: React.FC = () => {
           />
         </div>
 
-        <ResizableComponent resizeSide="right">
+        <ResizableComponent resizeSide="right" initialWidth={300}>
           <div className="size-full border-y-0 border-l-0 border-r-[0.001px] border-solid border-neutral-700">
             <SidebarManager />
           </div>

src/components/Sidebars/MainSidebar.tsx

@@ -14,7 +14,10 @@ const SidebarManager: React.FC = () => {
   const { sidebarShowing } = useChatContext()
 
   const [searchQuery, setSearchQuery] = useState<string>('')
-  const [searchResults, setSearchResults] = useState<DBQueryResult[]>([])
+  const [searchResults, setSearchResults] = useState<{ vectorResults: DBQueryResult[]; textResults: DBQueryResult[] }>({
+    vectorResults: [],
+    textResults: [],
+  })
 
   return (
     <div className="size-full overflow-y-hidden">

src/components/Sidebars/SearchComponent.tsx

@@ -1,16 +1,18 @@
-import React, { useEffect, useRef, useCallback } from 'react'
+import React, { useEffect, useRef, useCallback, useState } from 'react'
 import { DBQueryResult } from 'electron/main/vector-database/schema'
 import posthog from 'posthog-js'
 import { FaSearch } from 'react-icons/fa'
 import { debounce } from 'lodash'
 import { DBSearchPreview } from '../File/DBResultPreview'
 import { useContentContext } from '@/contexts/ContentContext'
 
+type SearchType = 'vector' | 'text' | 'hybrid'
+
 interface SearchComponentProps {
   searchQuery: string
   setSearchQuery: (query: string) => void
-  searchResults: DBQueryResult[]
-  setSearchResults: (results: DBQueryResult[]) => void
+  searchResults: { vectorResults: DBQueryResult[]; textResults: DBQueryResult[] }
+  setSearchResults: (results: { vectorResults: DBQueryResult[]; textResults: DBQueryResult[] }) => void
 }
 
 const SearchComponent: React.FC<SearchComponentProps> = ({
@@ -21,13 +23,14 @@ const SearchComponent: React.FC<SearchComponentProps> = ({
 }) => {
   const { openContent: openTabContent } = useContentContext()
   const searchInputRef = useRef<HTMLInputElement>(null)
+  const [searchType, setSearchType] = useState<SearchType>('vector')
 
   const handleSearch = useCallback(
     async (query: string) => {
-      const results: DBQueryResult[] = await window.database.search(query, 50)
+      const results = await window.database.multiModalSearch(query, 50, searchType)
       setSearchResults(results)
     },
-    [setSearchResults],
+    [setSearchResults, searchType],
   )
 
   const debouncedSearch = useCallback(
@@ -46,7 +49,7 @@ const SearchComponent: React.FC<SearchComponentProps> = ({
     if (searchQuery) {
       debouncedSearch(searchQuery)
     }
-  }, [searchQuery, debouncedSearch])
+  }, [searchQuery, debouncedSearch, searchType])
 
   const openFileSelectSearch = useCallback(
     (path: string) => {
@@ -70,13 +73,32 @@ const SearchComponent: React.FC<SearchComponentProps> = ({
           onChange={(e) => setSearchQuery(e.target.value)}
           placeholder="Semantic search..."
         />
+        <select
+          value={searchType}
+          onChange={(e) => setSearchType(e.target.value as SearchType)}
+          className="absolute right-3 top-1/2 -translate-y-1/2 rounded bg-neutral-700 text-white"
+        >
+          <option value="vector">Vector</option>
+          <option value="text">Text</option>
+          <option value="hybrid">Hybrid</option>
+        </select>
       </div>
       <div className="mt-2 w-full">
-        {searchResults.length > 0 && (
+        {searchResults?.textResults?.length > 0 && (
+          <div className="mt-4 w-full">
+            <h3 className="mb-2 text-white">Text Search Results</h3>
+            {searchResults.textResults.map((result, index) => (
+              // eslint-disable-next-line react/no-array-index-key
+              <DBSearchPreview key={`text-${index}`} dbResult={result} onSelect={openFileSelectSearch} />
+            ))}
+          </div>
+        )}
+        {searchResults?.vectorResults?.length > 0 && (
           <div className="w-full">
-            {searchResults.map((result, index) => (
+            <h3 className="mb-2 text-white">Vector Search Results</h3>
+            {searchResults.vectorResults.map((result, index) => (
               // eslint-disable-next-line react/no-array-index-key
-              <DBSearchPreview key={index} dbResult={result} onSelect={openFileSelectSearch} />
+              <DBSearchPreview key={`vector-${index}`} dbResult={result} onSelect={openFileSelectSearch} />
             ))}
           </div>
         )}