diff --git a/benchmarking/src/app/app.tsx b/benchmarking/src/app/app.tsx
index 35f39d2f..26980421 100644
--- a/benchmarking/src/app/app.tsx
+++ b/benchmarking/src/app/app.tsx
@@ -2,6 +2,7 @@ import { Link, Route, BrowserRouter as Router, Routes } from 'react-router-dom';
import { IndexedDBMProvider } from './dbm-context/indexed-dbm-context';
import { MemoryDBMProvider } from './dbm-context/memory-dbm-context';
import { NativeDBMProvider } from './dbm-context/native-dbm-context';
+import { OPFSDBMProvider } from './dbm-context/opfs-dbm-context';
import { ParallelIndexedDBMProvider } from './dbm-context/parallel-indexed-dbm-context';
import { ParallelMemoryDBMProvider } from './dbm-context/parallel-memory-dbm-context';
import { RawDBMProvider } from './dbm-context/raw-dbm-context';
@@ -23,6 +24,9 @@ export function App() {
IndexedDB DuckDB
+
+ OPFS DuckDB
+
Parallel Memory DuckDB
@@ -75,6 +79,19 @@ export function App() {
}
/>
+
+ OPFS DuckDB
+
+
+
+
+
+
+ }
+ />
{
+ const fileManagerRef = React.useRef(null);
+ const [dbm, setdbm] = useState(null);
+
+ const instanceManagerRef = React.useRef(
+ new InstanceManager({ path: 'opfs://meerkat.db' })
+ );
+
+ const dbState = useAsyncDuckDB();
+
+ useClassicEffect(() => {
+ if (!dbState) {
+ return;
+ }
+ fileManagerRef.current = new OPFSFileManager({
+ instanceManager: instanceManagerRef.current,
+ fetchTableFileBuffers: async (table) => {
+ return [];
+ },
+ logger: log,
+ onEvent: (event) => {
+ console.info(event);
+ },
+ });
+
+ log.setLevel('DEBUG');
+ const dbm = new DBM({
+ instanceManager: instanceManagerRef.current,
+ fileManager: fileManagerRef.current,
+ logger: log,
+ onEvent: (event) => {
+ console.info(event);
+ },
+ });
+ setdbm(dbm);
+ }, [dbState]);
+
+ if (!dbm || !fileManagerRef.current) {
+ return Loading...
;
+ }
+
+ return (
+
+ {children}
+
+ );
+};
diff --git a/benchmarking/src/app/file-loader/file-loader.tsx b/benchmarking/src/app/file-loader/file-loader.tsx
index 837647b2..8fa5146b 100644
--- a/benchmarking/src/app/file-loader/file-loader.tsx
+++ b/benchmarking/src/app/file-loader/file-loader.tsx
@@ -32,9 +32,17 @@ export const FileLoader = ({ children }: { children: JSX.Element }) => {
});
// Create views for raw and memory file manager after registering the files
- if (fileManagerType === 'raw' || fileManagerType === 'memory') {
- await dbm.query(generateViewQuery('taxi', ['taxi.parquet']));
- await dbm.query(generateViewQuery('taxi_json', ['taxi_json.parquet']));
+ if (
+ fileManagerType === 'raw' ||
+ fileManagerType === 'memory' ||
+ fileManagerType === 'opfs'
+ ) {
+ const useTable = fileManagerType === 'opfs';
+
+ await dbm.query(generateViewQuery('taxi', ['taxi.parquet'], useTable));
+ await dbm.query(
+ generateViewQuery('taxi_json', ['taxi_json.parquet'], useTable)
+ );
}
setIsFileLoader(true);
diff --git a/benchmarking/src/app/hooks/dbm-context.tsx b/benchmarking/src/app/hooks/dbm-context.tsx
index e6f7a38c..16536bce 100644
--- a/benchmarking/src/app/hooks/dbm-context.tsx
+++ b/benchmarking/src/app/hooks/dbm-context.tsx
@@ -13,6 +13,7 @@ export type DBMContextType = {
| 'raw'
| 'memory'
| 'indexdb'
+ | 'opfs'
| 'native'
| 'parallel-memory'
| 'parallel-indexdb';
diff --git a/benchmarking/src/app/query-benchmarking/query-benchmarking.tsx b/benchmarking/src/app/query-benchmarking/query-benchmarking.tsx
index b23da20d..b1c8a1c7 100644
--- a/benchmarking/src/app/query-benchmarking/query-benchmarking.tsx
+++ b/benchmarking/src/app/query-benchmarking/query-benchmarking.tsx
@@ -27,7 +27,10 @@ export const QueryBenchmarking = () => {
filePaths = table.files.map((file) => file.fileName);
}
- await dbm.query(generateViewQuery(table.tableName, filePaths));
+ const useTable = fileManagerType === 'opfs';
+ await dbm.query(
+ generateViewQuery(table.tableName, filePaths, useTable)
+ );
}
},
[fileManagerType, dbm]
diff --git a/benchmarking/src/app/utils.ts b/benchmarking/src/app/utils.ts
index 0e2521c9..1223a04a 100644
--- a/benchmarking/src/app/utils.ts
+++ b/benchmarking/src/app/utils.ts
@@ -1,5 +1,11 @@
-export const generateViewQuery = (tableName: string, files: string[]) => {
- return `CREATE VIEW IF NOT EXISTS ${tableName} AS SELECT * FROM read_parquet(['${files.join(
+export const generateViewQuery = (
+ tableName: string,
+ files: string[],
+ useTable = false
+) => {
+ const targetType = useTable ? 'TABLE' : 'VIEW';
+
+ return `CREATE ${targetType} IF NOT EXISTS ${tableName} AS SELECT * FROM read_parquet(['${files.join(
"','"
)}']);`;
};
diff --git a/meerkat-dbm/src/dbm/dbm.ts b/meerkat-dbm/src/dbm/dbm.ts
index 0dd9c81f..12eb3207 100644
--- a/meerkat-dbm/src/dbm/dbm.ts
+++ b/meerkat-dbm/src/dbm/dbm.ts
@@ -102,6 +102,7 @@ export class DBM extends TableLockManager {
private async _getConnection() {
if (!this.connection) {
const db = await this.instanceManager.getDB();
+
this.connection = await db.connect();
if (this.onCreateConnection) {
await this.onCreateConnection(this.connection);
diff --git a/meerkat-dbm/src/file-manager/index.ts b/meerkat-dbm/src/file-manager/index.ts
index dac55eef..50794a57 100644
--- a/meerkat-dbm/src/file-manager/index.ts
+++ b/meerkat-dbm/src/file-manager/index.ts
@@ -6,3 +6,4 @@ export * from './memory/memory-file-manager';
export * from './memory/parallel-memory-file-manager';
export * from './memory/runner-memory-file-manager';
export * from './native/native-file-manager';
+export * from './opfs/opfs-file-manager';
diff --git a/meerkat-dbm/src/file-manager/opfs/opfs-file-manager.ts b/meerkat-dbm/src/file-manager/opfs/opfs-file-manager.ts
new file mode 100644
index 00000000..d9bd0eb1
--- /dev/null
+++ b/meerkat-dbm/src/file-manager/opfs/opfs-file-manager.ts
@@ -0,0 +1,130 @@
+import { TableConfig } from '../../dbm/types';
+import { mergeFileStoreIntoTable } from '../../utils';
+import {
+ FileBufferStore,
+ FileManagerConstructorOptions,
+ FileManagerType,
+} from '../file-manager-type';
+import { FileRegisterer } from '../file-registerer';
+import { BaseIndexedDBFileManager } from '../indexed-db/base-indexed-db-file-manager';
+
+export class OPFSFileManager
+ extends BaseIndexedDBFileManager
+ implements FileManagerType
+{
+ private fileRegisterer: FileRegisterer;
+ private configurationOptions: FileManagerConstructorOptions['options'];
+
+ fetchTableFileBuffers: (tableName: string) => Promise;
+
+ constructor({
+ fetchTableFileBuffers,
+ instanceManager,
+ options,
+ logger,
+ onEvent,
+ }: FileManagerConstructorOptions) {
+ super({ instanceManager, fetchTableFileBuffers, logger, onEvent });
+
+ this.fetchTableFileBuffers = fetchTableFileBuffers;
+ this.fileRegisterer = new FileRegisterer({ instanceManager });
+ this.configurationOptions = options;
+ }
+
+ async bulkRegisterFileBuffer(fileBuffers: FileBufferStore[]): Promise {
+ const db = await this.instanceManager.getDB();
+
+ const tableNames = Array.from(
+ new Set(fileBuffers.map((fileBuffer) => fileBuffer.tableName))
+ );
+
+ const currentTableData = await this.indexedDB.tablesKey.toArray();
+
+ const updatedTableMap = mergeFileStoreIntoTable(
+ fileBuffers,
+ currentTableData
+ );
+
+ /**
+ * Extracts the tables and files data from the tablesMap and fileBuffers
+ * in format that can be stored in IndexedDB
+ */
+ const updatedTableData = tableNames.map((tableName) => {
+ return { tableName, files: updatedTableMap.get(tableName)?.files ?? [] };
+ });
+
+ const newFilesData = fileBuffers.map((fileBuffer) => {
+ return { buffer: fileBuffer.buffer, fileName: fileBuffer.fileName };
+ });
+
+ // Update the tables and files table in IndexedDB
+ await this.indexedDB
+ .transaction(
+ 'rw',
+ this.indexedDB.tablesKey,
+ this.indexedDB.files,
+ async () => {
+ await this.indexedDB.tablesKey.bulkPut(updatedTableData);
+
+ await Promise.all(
+ newFilesData.map((file) =>
+ this.fileRegisterer.registerFileBuffer(file.fileName, file.buffer)
+ )
+ );
+ }
+ )
+ .catch((error) => {
+ console.error(error);
+ });
+ }
+
+ async registerFileBuffer(fileBuffer: FileBufferStore): Promise {
+ const { buffer, fileName, tableName } = fileBuffer;
+
+ const currentTableData = await this.indexedDB.tablesKey.toArray();
+
+ const updatedTableMap = mergeFileStoreIntoTable(
+ [fileBuffer],
+ currentTableData
+ );
+
+ // Update the tables and files table in IndexedDB
+ await this.indexedDB
+ .transaction(
+ 'rw',
+ this.indexedDB.tablesKey,
+ this.indexedDB.files,
+ async () => {
+ await this.indexedDB.tablesKey.put({
+ tableName: fileBuffer.tableName,
+ files: updatedTableMap.get(tableName)?.files ?? [],
+ });
+
+ await this.fileRegisterer.registerFileBuffer(fileName, buffer);
+ }
+ )
+ .catch((error) => {
+ console.error(error);
+ });
+ }
+
+ async mountFileBufferByTables(tables: TableConfig[]): Promise {
+ const tableData = await this.getFilesNameForTables(tables);
+
+ /**
+ * Check if the file registered size is not more than the limit
+ * If it is more than the limit, then remove the files which are not needed while mounting this the tables
+ */
+ }
+
+ async dropFilesByTableName(
+ tableName: string,
+ fileNames: string[]
+ ): Promise {
+ const tableData = await this.indexedDB.tablesKey.get(tableName);
+ }
+
+ async onDBShutdownHandler() {
+ this.fileRegisterer.flushFileCache();
+ }
+}