diff --git a/benchmarking/src/app/app.tsx b/benchmarking/src/app/app.tsx index 35f39d2f..26980421 100644 --- a/benchmarking/src/app/app.tsx +++ b/benchmarking/src/app/app.tsx @@ -2,6 +2,7 @@ import { Link, Route, BrowserRouter as Router, Routes } from 'react-router-dom'; import { IndexedDBMProvider } from './dbm-context/indexed-dbm-context'; import { MemoryDBMProvider } from './dbm-context/memory-dbm-context'; import { NativeDBMProvider } from './dbm-context/native-dbm-context'; +import { OPFSDBMProvider } from './dbm-context/opfs-dbm-context'; import { ParallelIndexedDBMProvider } from './dbm-context/parallel-indexed-dbm-context'; import { ParallelMemoryDBMProvider } from './dbm-context/parallel-memory-dbm-context'; import { RawDBMProvider } from './dbm-context/raw-dbm-context'; @@ -23,6 +24,9 @@ export function App() {
  • IndexedDB DuckDB
  • +
  • + OPFS DuckDB +
  • Parallel Memory DuckDB
  • @@ -75,6 +79,19 @@ export function App() { } /> + +

    OPFS DuckDB

    + + + + + + + } + /> { + const fileManagerRef = React.useRef(null); + const [dbm, setdbm] = useState(null); + + const instanceManagerRef = React.useRef( + new InstanceManager({ path: 'opfs://meerkat.db' }) + ); + + const dbState = useAsyncDuckDB(); + + useClassicEffect(() => { + if (!dbState) { + return; + } + fileManagerRef.current = new OPFSFileManager({ + instanceManager: instanceManagerRef.current, + fetchTableFileBuffers: async (table) => { + return []; + }, + logger: log, + onEvent: (event) => { + console.info(event); + }, + }); + + log.setLevel('DEBUG'); + const dbm = new DBM({ + instanceManager: instanceManagerRef.current, + fileManager: fileManagerRef.current, + logger: log, + onEvent: (event) => { + console.info(event); + }, + }); + setdbm(dbm); + }, [dbState]); + + if (!dbm || !fileManagerRef.current) { + return
    Loading...
    ; + } + + return ( + + {children} + + ); +}; diff --git a/benchmarking/src/app/file-loader/file-loader.tsx b/benchmarking/src/app/file-loader/file-loader.tsx index 837647b2..8fa5146b 100644 --- a/benchmarking/src/app/file-loader/file-loader.tsx +++ b/benchmarking/src/app/file-loader/file-loader.tsx @@ -32,9 +32,17 @@ export const FileLoader = ({ children }: { children: JSX.Element }) => { }); // Create views for raw and memory file manager after registering the files - if (fileManagerType === 'raw' || fileManagerType === 'memory') { - await dbm.query(generateViewQuery('taxi', ['taxi.parquet'])); - await dbm.query(generateViewQuery('taxi_json', ['taxi_json.parquet'])); + if ( + fileManagerType === 'raw' || + fileManagerType === 'memory' || + fileManagerType === 'opfs' + ) { + const useTable = fileManagerType === 'opfs'; + + await dbm.query(generateViewQuery('taxi', ['taxi.parquet'], useTable)); + await dbm.query( + generateViewQuery('taxi_json', ['taxi_json.parquet'], useTable) + ); } setIsFileLoader(true); diff --git a/benchmarking/src/app/hooks/dbm-context.tsx b/benchmarking/src/app/hooks/dbm-context.tsx index e6f7a38c..16536bce 100644 --- a/benchmarking/src/app/hooks/dbm-context.tsx +++ b/benchmarking/src/app/hooks/dbm-context.tsx @@ -13,6 +13,7 @@ export type DBMContextType = { | 'raw' | 'memory' | 'indexdb' + | 'opfs' | 'native' | 'parallel-memory' | 'parallel-indexdb'; diff --git a/benchmarking/src/app/query-benchmarking/query-benchmarking.tsx b/benchmarking/src/app/query-benchmarking/query-benchmarking.tsx index b23da20d..b1c8a1c7 100644 --- a/benchmarking/src/app/query-benchmarking/query-benchmarking.tsx +++ b/benchmarking/src/app/query-benchmarking/query-benchmarking.tsx @@ -27,7 +27,10 @@ export const QueryBenchmarking = () => { filePaths = table.files.map((file) => file.fileName); } - await dbm.query(generateViewQuery(table.tableName, filePaths)); + const useTable = fileManagerType === 'opfs'; + await dbm.query( + generateViewQuery(table.tableName, filePaths, useTable) + ); } }, [fileManagerType, dbm] diff --git a/benchmarking/src/app/utils.ts b/benchmarking/src/app/utils.ts index 0e2521c9..1223a04a 100644 --- a/benchmarking/src/app/utils.ts +++ b/benchmarking/src/app/utils.ts @@ -1,5 +1,11 @@ -export const generateViewQuery = (tableName: string, files: string[]) => { - return `CREATE VIEW IF NOT EXISTS ${tableName} AS SELECT * FROM read_parquet(['${files.join( +export const generateViewQuery = ( + tableName: string, + files: string[], + useTable = false +) => { + const targetType = useTable ? 'TABLE' : 'VIEW'; + + return `CREATE ${targetType} IF NOT EXISTS ${tableName} AS SELECT * FROM read_parquet(['${files.join( "','" )}']);`; }; diff --git a/meerkat-dbm/src/dbm/dbm.ts b/meerkat-dbm/src/dbm/dbm.ts index 0dd9c81f..12eb3207 100644 --- a/meerkat-dbm/src/dbm/dbm.ts +++ b/meerkat-dbm/src/dbm/dbm.ts @@ -102,6 +102,7 @@ export class DBM extends TableLockManager { private async _getConnection() { if (!this.connection) { const db = await this.instanceManager.getDB(); + this.connection = await db.connect(); if (this.onCreateConnection) { await this.onCreateConnection(this.connection); diff --git a/meerkat-dbm/src/file-manager/index.ts b/meerkat-dbm/src/file-manager/index.ts index dac55eef..50794a57 100644 --- a/meerkat-dbm/src/file-manager/index.ts +++ b/meerkat-dbm/src/file-manager/index.ts @@ -6,3 +6,4 @@ export * from './memory/memory-file-manager'; export * from './memory/parallel-memory-file-manager'; export * from './memory/runner-memory-file-manager'; export * from './native/native-file-manager'; +export * from './opfs/opfs-file-manager'; diff --git a/meerkat-dbm/src/file-manager/opfs/opfs-file-manager.ts b/meerkat-dbm/src/file-manager/opfs/opfs-file-manager.ts new file mode 100644 index 00000000..d9bd0eb1 --- /dev/null +++ b/meerkat-dbm/src/file-manager/opfs/opfs-file-manager.ts @@ -0,0 +1,130 @@ +import { TableConfig } from '../../dbm/types'; +import { mergeFileStoreIntoTable } from '../../utils'; +import { + FileBufferStore, + FileManagerConstructorOptions, + FileManagerType, +} from '../file-manager-type'; +import { FileRegisterer } from '../file-registerer'; +import { BaseIndexedDBFileManager } from '../indexed-db/base-indexed-db-file-manager'; + +export class OPFSFileManager + extends BaseIndexedDBFileManager + implements FileManagerType +{ + private fileRegisterer: FileRegisterer; + private configurationOptions: FileManagerConstructorOptions['options']; + + fetchTableFileBuffers: (tableName: string) => Promise; + + constructor({ + fetchTableFileBuffers, + instanceManager, + options, + logger, + onEvent, + }: FileManagerConstructorOptions) { + super({ instanceManager, fetchTableFileBuffers, logger, onEvent }); + + this.fetchTableFileBuffers = fetchTableFileBuffers; + this.fileRegisterer = new FileRegisterer({ instanceManager }); + this.configurationOptions = options; + } + + async bulkRegisterFileBuffer(fileBuffers: FileBufferStore[]): Promise { + const db = await this.instanceManager.getDB(); + + const tableNames = Array.from( + new Set(fileBuffers.map((fileBuffer) => fileBuffer.tableName)) + ); + + const currentTableData = await this.indexedDB.tablesKey.toArray(); + + const updatedTableMap = mergeFileStoreIntoTable( + fileBuffers, + currentTableData + ); + + /** + * Extracts the tables and files data from the tablesMap and fileBuffers + * in format that can be stored in IndexedDB + */ + const updatedTableData = tableNames.map((tableName) => { + return { tableName, files: updatedTableMap.get(tableName)?.files ?? [] }; + }); + + const newFilesData = fileBuffers.map((fileBuffer) => { + return { buffer: fileBuffer.buffer, fileName: fileBuffer.fileName }; + }); + + // Update the tables and files table in IndexedDB + await this.indexedDB + .transaction( + 'rw', + this.indexedDB.tablesKey, + this.indexedDB.files, + async () => { + await this.indexedDB.tablesKey.bulkPut(updatedTableData); + + await Promise.all( + newFilesData.map((file) => + this.fileRegisterer.registerFileBuffer(file.fileName, file.buffer) + ) + ); + } + ) + .catch((error) => { + console.error(error); + }); + } + + async registerFileBuffer(fileBuffer: FileBufferStore): Promise { + const { buffer, fileName, tableName } = fileBuffer; + + const currentTableData = await this.indexedDB.tablesKey.toArray(); + + const updatedTableMap = mergeFileStoreIntoTable( + [fileBuffer], + currentTableData + ); + + // Update the tables and files table in IndexedDB + await this.indexedDB + .transaction( + 'rw', + this.indexedDB.tablesKey, + this.indexedDB.files, + async () => { + await this.indexedDB.tablesKey.put({ + tableName: fileBuffer.tableName, + files: updatedTableMap.get(tableName)?.files ?? [], + }); + + await this.fileRegisterer.registerFileBuffer(fileName, buffer); + } + ) + .catch((error) => { + console.error(error); + }); + } + + async mountFileBufferByTables(tables: TableConfig[]): Promise { + const tableData = await this.getFilesNameForTables(tables); + + /** + * Check if the file registered size is not more than the limit + * If it is more than the limit, then remove the files which are not needed while mounting this the tables + */ + } + + async dropFilesByTableName( + tableName: string, + fileNames: string[] + ): Promise { + const tableData = await this.indexedDB.tablesKey.get(tableName); + } + + async onDBShutdownHandler() { + this.fileRegisterer.flushFileCache(); + } +}