From 59bc1778395926759b0453fe99321bbc8b8cc67a Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Tue, 23 Apr 2024 23:58:34 +0200 Subject: [PATCH 1/7] Add sql.js fallback for sqlite in wasm --- sqlite/lib/SQLiteService.js | 13 ++++-- sqlite/lib/sql.js.js | 84 +++++++++++++++++++++++++++++++++++++ sqlite/package.json | 5 ++- 3 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 sqlite/lib/sql.js.js diff --git a/sqlite/lib/SQLiteService.js b/sqlite/lib/SQLiteService.js index 2320b6e7e..f11354562 100644 --- a/sqlite/lib/SQLiteService.js +++ b/sqlite/lib/SQLiteService.js @@ -1,6 +1,12 @@ const { SQLService } = require('@cap-js/db-service') const cds = require('@sap/cds/lib') -const sqlite = require('better-sqlite3') +let sqlite +try { + sqlite = require('better-sqlite3') +} catch (err) { + // When failing to load better-sqlite3 it fallsback to sql.js (wasm version of sqlite) + sqlite = require('./sql.js.js') +} const $session = Symbol('dbc.session') const convStrm = require('stream/consumers') const { Readable } = require('stream') @@ -20,9 +26,10 @@ class SQLiteService extends SQLService { get factory() { return { options: { max: 1, ...this.options.pool }, - create: tenant => { + create: async tenant => { const database = this.url4(tenant) const dbc = new sqlite(database) + await dbc.ready const deterministic = { deterministic: true } dbc.function('session_context', key => dbc[$session][key]) @@ -224,7 +231,7 @@ class SQLiteService extends SQLService { // int64 is stored as native int64 for best comparison // Reading int64 as string to not loose precision Int64: expr => `CAST(${expr} as TEXT)`, - + // Reading decimal as string to not loose precision Decimal: expr => `CAST(${expr} as TEXT)`, diff --git a/sqlite/lib/sql.js.js b/sqlite/lib/sql.js.js new file mode 100644 index 000000000..d62626524 --- /dev/null +++ b/sqlite/lib/sql.js.js @@ -0,0 +1,84 @@ +const initSqlJs = require('sql.js'); + +const init = initSqlJs({}) + +class WasmSqlite { + constructor(database) { + // TODO: load / store database file contents + this.ready = init + .then(SQL => { this.db = new SQL.Database() }) + + this.memory = true + this.gc = new FinalizationRegistry(stmt => { stmt.free() }) + } + + prepare(sql) { + const stmt = this.db.prepare(sql) + const ret = { + run: (params) => { + try { + stmt.bind(params) + stmt.step() + return { changes: this.db.getRowsModified(stmt) } + } catch (err) { + if (err.message.indexOf('NOT NULL constraint failed:') === 0) { + err.code = 'SQLITE_CONSTRAINT_NOTNULL' + } + throw err + } + }, + get: (params) => { + const columns = stmt.getColumnNames() + stmt.bind(params) + stmt.step() + const row = stmt.get() + const ret = {} + for (let i = 0; i < columns.length; i++) { + ret[columns[i]] = row[i] + } + return ret + }, + all: (params) => { + const columns = stmt.getColumnNames() + const ret = [] + stmt.bind(params) + while (stmt.step()) { + const row = stmt.get() + const obj = {} + for (let i = 0; i < columns.length; i++) { + obj[columns[i]] = row[i] + } + ret.push(obj) + } + return ret + } + } + this.gc.register(ret, stmt) + return ret + } + + exec(sql) { + try { + const { columns, values } = this.db.exec(sql) + return !Array.isArray(values) ? values : values.map(val => { + const ret = {} + for (let i = 0; i < columns.length; i++) { + ret[columns[i]] = val[i] + } + return ret + }) + } catch (err) { + // REVISIT: address transaction errors + if (sql === 'BEGIN' || sql === 'ROLLBACK') { return } + throw err + } + } + + function(name, config, func) { + this.db.create_function(name, func || config) + } + + close() { this.db.close() } +} + +module.exports = WasmSqlite diff --git a/sqlite/package.json b/sqlite/package.json index cde98fc78..0c8a69797 100644 --- a/sqlite/package.json +++ b/sqlite/package.json @@ -33,6 +33,9 @@ "@cap-js/db-service": "^1.7.0", "better-sqlite3": "^9.3.0" }, + "optionalDependencies": { + "sql.js": "^1.10.3" + }, "peerDependencies": { "@sap/cds": ">=7.6" }, @@ -52,4 +55,4 @@ } }, "license": "SEE LICENSE" -} +} \ No newline at end of file From f456747ceb77e3485aac851596046a7af0713096 Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 24 Apr 2024 08:56:17 +0200 Subject: [PATCH 2/7] Update package.json --- package-lock.json | 9 +++++++++ sqlite/package.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 87d00ebad..fbe838b9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6064,6 +6064,12 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "node_modules/sql.js": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.10.3.tgz", + "integrity": "sha512-H46aWtQkdyjZwFQgraUruy5h/DyJBbAK3EA/WEMqiqF6PGPfKBSKBj/er3dVyYqVIoYfRf5TFM/loEjtQIrqJg==", + "optional": true + }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -6619,6 +6625,9 @@ "node": ">=16", "npm": ">=8" }, + "optionalDependencies": { + "sql.js": "^1.10.3" + }, "peerDependencies": { "@sap/cds": ">=7.6" } diff --git a/sqlite/package.json b/sqlite/package.json index 0c8a69797..6f411b396 100644 --- a/sqlite/package.json +++ b/sqlite/package.json @@ -55,4 +55,4 @@ } }, "license": "SEE LICENSE" -} \ No newline at end of file +} From 3a56fe172cb78b00f2a6ddcf9cfa5371b562bf04 Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 24 Apr 2024 09:03:02 +0200 Subject: [PATCH 3/7] Adding sql.js test pipeline --- .github/workflows/sqlite-wasm.yml | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/sqlite-wasm.yml diff --git a/.github/workflows/sqlite-wasm.yml b/.github/workflows/sqlite-wasm.yml new file mode 100644 index 000000000..62ba3b0f0 --- /dev/null +++ b/.github/workflows/sqlite-wasm.yml @@ -0,0 +1,37 @@ +name: Tests WASM + +on: + push: + branches: [main] + pull_request: + types: [opened, synchronize, reopened, auto_merge_enabled] + +# Allow parallel jobs on `main`, so that each commit is tested. For PRs, run only the latest commit. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 5 + name: Node.js ${{ matrix.node }} + + strategy: + fail-fast: true + matrix: + node: [18] + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: 'npm' + + - run: npm ci + # Remove better-sqlite3 to force switching to sql.js + - run: npm install sql.js && rm -rf /node_modules/better-sqlite3/ + - run: npm test -w sqlite -- --maxWorkers=1 + env: + FORCE_COLOR: true From e44a447a039a148f04d26515e66d4d238630526a Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 24 Apr 2024 09:08:14 +0200 Subject: [PATCH 4/7] Double checking that the fallback is tested --- sqlite/lib/SQLiteService.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sqlite/lib/SQLiteService.js b/sqlite/lib/SQLiteService.js index f11354562..79c9507f8 100644 --- a/sqlite/lib/SQLiteService.js +++ b/sqlite/lib/SQLiteService.js @@ -3,7 +3,9 @@ const cds = require('@sap/cds/lib') let sqlite try { sqlite = require('better-sqlite3') + process.stdout.write('Using default sqlite driver better-sqlite3\n') } catch (err) { + process.stdout.write('Using fallback sqlite driver sql.js\n') // When failing to load better-sqlite3 it fallsback to sql.js (wasm version of sqlite) sqlite = require('./sql.js.js') } From 9b1281a12cf05ece6c965220227a3b52d864a7f2 Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 24 Apr 2024 09:09:30 +0200 Subject: [PATCH 5/7] Adjust better-sqlite3 removal --- .github/workflows/sqlite-wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sqlite-wasm.yml b/.github/workflows/sqlite-wasm.yml index 62ba3b0f0..1fca0ff55 100644 --- a/.github/workflows/sqlite-wasm.yml +++ b/.github/workflows/sqlite-wasm.yml @@ -31,7 +31,7 @@ jobs: - run: npm ci # Remove better-sqlite3 to force switching to sql.js - - run: npm install sql.js && rm -rf /node_modules/better-sqlite3/ + - run: npm install sql.js && rm -rf node_modules/better-sqlite3/ - run: npm test -w sqlite -- --maxWorkers=1 env: FORCE_COLOR: true From 3ebf2f4657f05901f22037999bd72728156c3bef Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 24 Apr 2024 09:11:08 +0200 Subject: [PATCH 6/7] Remove debug logging --- sqlite/lib/SQLiteService.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/sqlite/lib/SQLiteService.js b/sqlite/lib/SQLiteService.js index 79c9507f8..f11354562 100644 --- a/sqlite/lib/SQLiteService.js +++ b/sqlite/lib/SQLiteService.js @@ -3,9 +3,7 @@ const cds = require('@sap/cds/lib') let sqlite try { sqlite = require('better-sqlite3') - process.stdout.write('Using default sqlite driver better-sqlite3\n') } catch (err) { - process.stdout.write('Using fallback sqlite driver sql.js\n') // When failing to load better-sqlite3 it fallsback to sql.js (wasm version of sqlite) sqlite = require('./sql.js.js') } From 6546a7d373987e5ac6473b6b4ce202f2bdc9816e Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Tue, 28 May 2024 10:48:15 +0200 Subject: [PATCH 7/7] Remove linting warning --- sqlite/lib/sql.js.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite/lib/sql.js.js b/sqlite/lib/sql.js.js index d62626524..3fb1119ac 100644 --- a/sqlite/lib/sql.js.js +++ b/sqlite/lib/sql.js.js @@ -3,7 +3,7 @@ const initSqlJs = require('sql.js'); const init = initSqlJs({}) class WasmSqlite { - constructor(database) { + constructor(/*database*/) { // TODO: load / store database file contents this.ready = init .then(SQL => { this.db = new SQL.Database() })