From a05b11a0fe4af269ceb14ab20e167de9ce4104c6 Mon Sep 17 00:00:00 2001 From: MicroBlock <66859419+MicroCBer@users.noreply.github.com> Date: Tue, 13 Aug 2024 19:41:15 +0800 Subject: [PATCH 1/5] feat: implement node-sqlite --- packages/node-sqlite/.npmignore | 2 + packages/node-sqlite/package.json | 49 +++ packages/node-sqlite/readme.md | 3 + packages/node-sqlite/src/builder.ts | 113 ++++++ packages/node-sqlite/src/index.ts | 430 +++++++++++++++++++++ packages/node-sqlite/src/locales/en-US.yml | 1 + packages/node-sqlite/src/locales/zh-CN.yml | 1 + packages/node-sqlite/src/node-sqlite.d.ts | 42 ++ packages/node-sqlite/tests/index.spec.ts | 32 ++ packages/node-sqlite/tsconfig.json | 10 + yakumo.yml | 14 +- 11 files changed, 693 insertions(+), 4 deletions(-) create mode 100644 packages/node-sqlite/.npmignore create mode 100644 packages/node-sqlite/package.json create mode 100644 packages/node-sqlite/readme.md create mode 100644 packages/node-sqlite/src/builder.ts create mode 100644 packages/node-sqlite/src/index.ts create mode 100644 packages/node-sqlite/src/locales/en-US.yml create mode 100644 packages/node-sqlite/src/locales/zh-CN.yml create mode 100644 packages/node-sqlite/src/node-sqlite.d.ts create mode 100644 packages/node-sqlite/tests/index.spec.ts create mode 100644 packages/node-sqlite/tsconfig.json diff --git a/packages/node-sqlite/.npmignore b/packages/node-sqlite/.npmignore new file mode 100644 index 00000000..7e5fcbc1 --- /dev/null +++ b/packages/node-sqlite/.npmignore @@ -0,0 +1,2 @@ +.DS_Store +tsconfig.tsbuildinfo diff --git a/packages/node-sqlite/package.json b/packages/node-sqlite/package.json new file mode 100644 index 00000000..d641781e --- /dev/null +++ b/packages/node-sqlite/package.json @@ -0,0 +1,49 @@ +{ + "name": "@minatojs/driver-node-sqlite", + "version": "4.4.1", + "description": "node:sqlite Driver for Minato", + "type": "module", + "main": "lib/index.cjs", + "module": "lib/index.mjs", + "typings": "lib/index.d.ts", + "exports": { + ".": { + "import": "./lib/index.mjs", + "require": "./lib/index.cjs", + "types": "./lib/index.d.ts" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib", + "src" + ], + "author": "Shigma ", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/shigma/minato.git", + "directory": "packages/sqlite" + }, + "bugs": { + "url": "https://github.com/shigma/minato/issues" + }, + "homepage": "https://github.com/shigma/minato/packages/sqlite#readme", + "keywords": [ + "orm", + "database", + "driver", + "sqlite" + ], + "peerDependencies": { + "minato": "^3.4.3" + }, + "devDependencies": { + "@minatojs/tests": "^2.4.1" + }, + "dependencies": { + "@minatojs/sql-utils": "^5.4.1", + "cosmokit": "^1.6.2" + } +} diff --git a/packages/node-sqlite/readme.md b/packages/node-sqlite/readme.md new file mode 100644 index 00000000..d40786df --- /dev/null +++ b/packages/node-sqlite/readme.md @@ -0,0 +1,3 @@ +# @minatojs/driver-node-sqlite + +node:sqlite Driver for Minato. diff --git a/packages/node-sqlite/src/builder.ts b/packages/node-sqlite/src/builder.ts new file mode 100644 index 00000000..0376beae --- /dev/null +++ b/packages/node-sqlite/src/builder.ts @@ -0,0 +1,113 @@ +import { Builder, escapeId } from '@minatojs/sql-utils' +import { Binary, Dict, isNullable } from 'cosmokit' +import { Driver, Field, Model, randomId, RegExpLike, Type } from 'minato' + +export class SQLiteBuilder extends Builder { + protected escapeMap = { + "'": "''", + } + + constructor(protected driver: Driver, tables?: Dict) { + super(driver, tables) + + this.queryOperators.$regexFor = (key, value) => typeof value === 'string' ? `${this.escape(value)} regexp ${key}` + : value.flags?.includes('i') ? `regexp2(${key}, ${this.escape(value.input)}, 'i')` + : `${this.escape(value.input)} regexp ${key}` + + this.evalOperators.$if = (args) => `iif(${args.map(arg => this.parseEval(arg)).join(', ')})` + this.evalOperators.$regex = ([key, value, flags]) => (flags?.includes('i') || (value instanceof RegExp && value.flags?.includes('i'))) + ? `regexp2(${this.parseEval(value)}, ${this.parseEval(key)}, ${this.escape(flags ?? (value as any).flags)})` + : `regexp(${this.parseEval(value)}, ${this.parseEval(key)})` + + this.evalOperators.$concat = (args) => `(${args.map(arg => this.parseEval(arg)).join('||')})` + this.evalOperators.$modulo = ([left, right]) => `modulo(${this.parseEval(left)}, ${this.parseEval(right)})` + this.evalOperators.$log = ([left, right]) => isNullable(right) + ? `log(${this.parseEval(left)})` + : `log(${this.parseEval(left)}) / log(${this.parseEval(right)})` + this.evalOperators.$length = (expr) => this.createAggr(expr, value => `count(${value})`, value => this.isEncoded() ? this.jsonLength(value) + : this.asEncoded(`iif(${value}, LENGTH(${value}) - LENGTH(REPLACE(${value}, ${this.escape(',')}, ${this.escape('')})) + 1, 0)`, false)) + this.evalOperators.$number = (arg) => { + const type = Type.fromTerm(arg) + const value = this.parseEval(arg) + const res = Field.date.includes(type.type as any) ? `cast(${value} / 1000 as integer)` : `cast(${this.parseEval(arg)} as double)` + return this.asEncoded(`ifnull(${res}, 0)`, false) + } + + const binaryXor = (left: string, right: string) => `((${left} & ~${right}) | (~${left} & ${right}))` + this.evalOperators.$xor = (args) => { + const type = Type.fromTerm(this.state.expr, Type.Boolean) + if (Field.boolean.includes(type.type)) return args.map(arg => this.parseEval(arg)).reduce((prev, curr) => `(${prev} != ${curr})`) + else return args.map(arg => this.parseEval(arg)).reduce((prev, curr) => binaryXor(prev, curr)) + } + + this.transformers['bigint'] = { + encode: value => `cast(${value} as text)`, + decode: value => `cast(${value} as integer)`, + load: value => isNullable(value) ? value : BigInt(value), + dump: value => isNullable(value) ? value : `${value}`, + } + + this.transformers['binary'] = { + encode: value => `hex(${value})`, + decode: value => `unhex(${value})`, + load: value => isNullable(value) || typeof value === 'object' ? value : Binary.fromHex(value), + dump: value => isNullable(value) || typeof value === 'string' ? value : Binary.toHex(value), + } + } + + escapePrimitive(value: any, type?: Type) { + if (value instanceof Date) value = +value + else if (value instanceof RegExp) value = value.source + else if (Binary.is(value)) return `X'${Binary.toHex(value)}'` + else if (Binary.isSource(value)) return `X'${Binary.toHex(Binary.fromSource(value))}'` + return super.escapePrimitive(value, type) + } + + protected createElementQuery(key: string, value: any) { + if (this.isJsonQuery(key)) { + return this.jsonContains(key, this.escape(value, 'json')) + } else { + return `(',' || ${key} || ',') LIKE ${this.escape('%,' + value + ',%')}` + } + } + + protected createRegExpQuery(key: string, value: string | RegExpLike) { + if (typeof value !== 'string' && value.flags?.includes('i')) { + return `regexp2(${this.escape(typeof value === 'string' ? value : value.source)}, ${key}, ${this.escape(value.flags)})` + } else { + return `regexp(${this.escape(typeof value === 'string' ? value : value.source)}, ${key})` + } + } + + protected jsonLength(value: string) { + return this.asEncoded(`json_array_length(${value})`, false) + } + + protected jsonContains(obj: string, value: string) { + return this.asEncoded(`json_array_contains(${obj}, ${value})`, false) + } + + protected encode(value: string, encoded: boolean, pure: boolean = false, type?: Type) { + return encoded ? super.encode(value, encoded, pure, type) + : (encoded === this.isEncoded() && !pure) ? value + : this.asEncoded(this.transform(`(${value} ->> '$')`, type, 'decode'), pure ? undefined : false) + } + + protected createAggr(expr: any, aggr: (value: string) => string, nonaggr?: (value: string) => string) { + if (!this.state.group && !nonaggr) { + const value = this.parseEval(expr, false) + return `(select ${aggr(escapeId('value'))} from json_each(${value}) ${randomId()})` + } else { + return super.createAggr(expr, aggr, nonaggr) + } + } + + protected groupArray(value: string) { + const res = this.isEncoded() ? `('[' || group_concat(${value}) || ']')` : `('[' || group_concat(json_quote(${value})) || ']')` + return this.asEncoded(`ifnull(${res}, json_array())`, true) + } + + protected transformJsonField(obj: string, path: string) { + return this.asEncoded(`(${obj} -> '$${path}')`, true) + } +} diff --git a/packages/node-sqlite/src/index.ts b/packages/node-sqlite/src/index.ts new file mode 100644 index 00000000..519b5bf0 --- /dev/null +++ b/packages/node-sqlite/src/index.ts @@ -0,0 +1,430 @@ +import { Binary, deepEqual, Dict, difference, isNullable, makeArray, mapValues } from 'cosmokit' +import { Driver, Eval, executeUpdate, Field, getCell, hasSubquery, isEvalExpr, Selection, z } from 'minato' +import { escapeId } from '@minatojs/sql-utils' +import { resolve } from 'node:path' +import type { DatabaseSync, StatementSync } from 'node:sqlite' +import enUS from './locales/en-US.yml' +import zhCN from './locales/zh-CN.yml' +import { SQLiteBuilder } from './builder' + +function getTypeDef({ deftype: type }: Field) { + switch (type) { + case 'primary': + case 'boolean': + case 'integer': + case 'unsigned': + case 'bigint': + case 'date': + case 'time': + case 'timestamp': return `INTEGER` + case 'float': + case 'double': + case 'decimal': return `REAL` + case 'char': + case 'string': + case 'text': + case 'list': + case 'json': return `TEXT` + case 'binary': return `BLOB` + default: throw new Error(`unsupported type: ${type}`) + } +} + +export interface SQLiteFieldInfo { + cid: number + name: string + type: string + notnull: number + dflt_value: string + pk: boolean +} + +export class SQLiteDriver extends Driver { + static name = 'sqlite' + + path!: string + db!: DatabaseSync + sql = new SQLiteBuilder(this) + beforeUnload?: () => void + + private _transactionTask?: Promise + + /** synchronize table schema */ + async prepare(table: string, dropKeys?: string[]) { + const columns = this._all(`PRAGMA table_info(${escapeId(table)})`) as SQLiteFieldInfo[] + const model = this.model(table) + const columnDefs: string[] = [] + const indexDefs: string[] = [] + const alter: string[] = [] + const mapping: Dict = {} + let shouldMigrate = false + + // field definitions + for (const key in model.fields) { + if (!Field.available(model.fields[key])) { + if (dropKeys?.includes(key)) shouldMigrate = true + continue + } + + const legacy = [key, ...model.fields[key]!.legacy || []] + const column = columns.find(({ name }) => legacy.includes(name)) + const { initial, nullable = true } = model.fields[key]! + const typedef = getTypeDef(model.fields[key]!) + let def = `${escapeId(key)} ${typedef}` + if (key === model.primary && model.autoInc) { + def += ' NOT NULL PRIMARY KEY AUTOINCREMENT' + } else { + def += (nullable ? ' ' : ' NOT ') + 'NULL' + if (!isNullable(initial)) { + def += ' DEFAULT ' + this.sql.escape(this.sql.dump({ [key]: initial }, model)[key]) + } + } + columnDefs.push(def) + if (!column) { + alter.push('ADD ' + def) + } else { + mapping[column.name] = key + shouldMigrate ||= column.name !== key || column.type !== typedef + } + } + + // index definitions + if (model.primary && !model.autoInc) { + indexDefs.push(`PRIMARY KEY (${this._joinKeys(makeArray(model.primary))})`) + } + if (model.unique) { + indexDefs.push(...model.unique.map(keys => `UNIQUE (${this._joinKeys(makeArray(keys))})`)) + } + if (model.foreign) { + indexDefs.push(...Object.entries(model.foreign).map(([key, value]) => { + const [table, key2] = value! + return `FOREIGN KEY (\`${key}\`) REFERENCES ${escapeId(table)} (\`${key2}\`)` + })) + } + + if (!columns.length) { + this.logger.info('auto creating table %c', table) + this._run(`CREATE TABLE ${escapeId(table)} (${[...columnDefs, ...indexDefs].join(', ')})`) + } else if (shouldMigrate) { + // preserve old columns + for (const { name, type, notnull, pk, dflt_value: value } of columns) { + if (mapping[name] || dropKeys?.includes(name)) continue + let def = `${escapeId(name)} ${type}` + def += (notnull ? ' NOT ' : ' ') + 'NULL' + if (pk) def += ' PRIMARY KEY' + if (value !== null) def += ' DEFAULT ' + this.sql.escape(value) + columnDefs.push(def) + mapping[name] = name + } + + const temp = table + '_temp' + const fields = Object.keys(mapping).map(escapeId).join(', ') + this.logger.info('auto migrating table %c', table) + this._run(`CREATE TABLE ${escapeId(temp)} (${[...columnDefs, ...indexDefs].join(', ')})`) + try { + this._run(`INSERT INTO ${escapeId(temp)} SELECT ${fields} FROM ${escapeId(table)}`) + this._run(`DROP TABLE ${escapeId(table)}`) + } catch (error) { + this._run(`DROP TABLE ${escapeId(temp)}`) + throw error + } + this._run(`ALTER TABLE ${escapeId(temp)} RENAME TO ${escapeId(table)}`) + } else if (alter.length) { + this.logger.info('auto updating table %c', table) + for (const def of alter) { + this._run(`ALTER TABLE ${escapeId(table)} ${def}`) + } + } + + if (dropKeys) return + dropKeys = [] + await this.migrate(table, { + error: this.logger.warn, + before: keys => keys.every(key => columns.some(({ name }) => name === key)), + after: keys => dropKeys!.push(...keys), + finalize: () => { + if (!dropKeys!.length) return + this.prepare(table, dropKeys) + }, + }) + } + + async start() { + this.path = this.config.path + if (this.path !== ':memory:') { + this.path = resolve(this.ctx.baseDir, this.path) + } + const isBrowser = process.env.KOISHI_ENV === 'browser' + if (isBrowser) { + throw new Error('node:sqlite driver is not supported in browser environment') + } + + const DatabaseSync = await import('node:sqlite').then((m) => m.DatabaseSync) + .catch(e => { + if (e.toString().includes('ERR_UNKNOWN_BUILTIN_MODULE')) { + throw new Error('The sqlite3 module is currently experimental. You have to install Node.JS 22.5+ and run it with --experimental-sqlite to use it.') + } else { + throw e + } + }) + + this.db = new DatabaseSync(this.path) + // TODO: implement create function after https://github.com/nodejs/node/issues/54349 is resolved + // this.db.create_function('regexp', (pattern, str) => +new RegExp(pattern).test(str)) + // this.db.create_function('regexp2', (pattern, str, flags) => +new RegExp(pattern, flags).test(str)) + // this.db.create_function('json_array_contains', (array, value) => +(JSON.parse(array) as any[]).includes(JSON.parse(value))) + // this.db.create_function('modulo', (left, right) => left % right) + // this.db.create_function('rand', () => Math.random()) + + this.define({ + types: ['boolean'], + dump: value => isNullable(value) ? value : +value, + load: (value) => isNullable(value) ? value : !!value, + }) + + this.define({ + types: ['json'], + dump: value => JSON.stringify(value), + load: value => typeof value === 'string' ? JSON.parse(value) : value, + }) + + this.define({ + types: ['list'], + dump: value => Array.isArray(value) ? value.join(',') : value, + load: value => value ? value.split(',') : [], + }) + + this.define({ + types: ['date', 'time', 'timestamp'], + dump: value => isNullable(value) ? value as any : +new Date(value), + load: value => isNullable(value) ? value : new Date(Number(value)), + }) + + this.define({ + types: ['binary'], + dump: value => isNullable(value) ? value : new Uint8Array(value), + load: value => isNullable(value) ? value : Binary.fromSource(value), + }) + + this.define({ + types: Field.number as any, + dump: value => value, + load: value => isNullable(value) ? value : Number(value), + }) + } + + _joinKeys(keys?: string[]) { + return keys?.length ? keys.map(key => `\`${key}\``).join(', ') : '*' + } + + async stop() { + await new Promise(resolve => setTimeout(resolve, 0)) + this.db?.close() + if (this.beforeUnload) { + this.beforeUnload() + window.removeEventListener('beforeunload', this.beforeUnload) + } + } + + _exec(sql: string, params: any, callback: (stmt: StatementSync) => any) { + try { + const stmt = this.db.prepare(sql) + const result = callback(stmt) + this.logger.debug('> %s', sql, params) + return result + } catch (e) { + this.logger.warn('> %s', sql, params) + throw e + } + } + + _all(sql: string, params: any = [], config?: { useBigInt: boolean }) { + return this._exec(sql, params, (stmt) => { + stmt.setReadBigInts(config?.useBigInt || false) + return stmt.all(...params) + }) + } + + _get(sql: string, params: any = [], config?: { useBigInt: boolean }) { + // @ts-ignore + return this._exec(sql, params, stmt => { + stmt.setReadBigInts(config?.useBigInt || false) + return stmt.get(...params) + }) + } + + _run(sql: string, params: any = [], callback?: () => any) { + this._exec(sql, params, stmt => stmt.run(...params)) + const result = callback?.() + return result + } + + async drop(table: string) { + this._run(`DROP TABLE ${escapeId(table)}`) + } + + async dropAll() { + const tables = Object.keys(this.database.tables) + for (const table of tables) { + this._run(`DROP TABLE ${escapeId(table)}`) + } + } + + async stats() { + // TODO: properly estimate size + const stats: Driver.Stats = { size: 100, tables: {} } + const tableNames: { name: string }[] = this._all('SELECT name FROM sqlite_master WHERE type="table" ORDER BY name;') + const dbstats: { name: string; size: number }[] = this._all('SELECT name, pgsize as size FROM "dbstat" WHERE aggregate=TRUE;') + tableNames.forEach(tbl => { + stats.tables[tbl.name] = this._get(`SELECT COUNT(*) as count FROM ${escapeId(tbl.name)};`) + stats.tables[tbl.name].size = dbstats.find(o => o.name === tbl.name)!.size + }) + return stats + } + + async remove(sel: Selection.Mutable) { + const { query, table, tables } = sel + const builder = new SQLiteBuilder(this, tables) + const filter = builder.parseQuery(query) + if (filter === '0') return {} + const result = this._run(`DELETE FROM ${escapeId(table)} WHERE ${filter}`, [], () => this._get(`SELECT changes() AS count`)) + return { matched: result.count, removed: result.count } + } + + async get(sel: Selection.Immutable) { + const { model, tables } = sel + const builder = new SQLiteBuilder(this, tables) + const sql = builder.get(sel) + if (!sql) return [] + const rows: any[] = this._all(sql, [], { useBigInt: true }) + return rows.map(row => builder.load(row, model)) + } + + async eval(sel: Selection.Immutable, expr: Eval.Expr) { + const builder = new SQLiteBuilder(this, sel.tables) + const inner = builder.get(sel.table as Selection, true, true) + const output = builder.parseEval(expr, false) + const { value } = this._get(`SELECT ${output} AS value FROM ${inner}`, [], { useBigInt: true }) + return builder.load(value, expr) + } + + _update(sel: Selection.Mutable, indexFields: string[], updateFields: string[], update: {}, data: {}) { + const { ref, table, tables, model } = sel + const builder = new SQLiteBuilder(this, tables) + executeUpdate(data, update, ref) + const row = builder.dump(data, model) + const assignment = updateFields.map((key) => `${escapeId(key)} = ?`).join(',') + const query = Object.fromEntries(indexFields.map(key => [key, row[key]])) + const filter = builder.parseQuery(query) + this._run(`UPDATE ${escapeId(table)} SET ${assignment} WHERE ${filter}`, updateFields.map((key) => row[key] ?? null)) + } + + async set(sel: Selection.Mutable, update: {}) { + const { model, table, query } = sel + const { primary } = model, fields = model.avaiableFields() + const updateFields = [...new Set(Object.keys(update).map((key) => { + return Object.keys(fields).find(field => field === key || key.startsWith(field + '.'))! + }))] + const primaryFields = makeArray(primary) + if (query.$expr || hasSubquery(sel.query) || Object.values(update).some(x => hasSubquery(x))) { + const sel2 = this.database.select(table as never, query) + sel2.tables[sel.ref] = sel2.tables[sel2.ref] + delete sel2.tables[sel2.ref] + sel2.ref = sel.ref + const project = mapValues(update as any, (value, key) => () => (isEvalExpr(value) ? value : Eval.literal(value, model.getType(key)))) + const rawUpsert = await sel2.project({ + ...project, + // do not touch sel2.row since it is not patched + ...Object.fromEntries(primaryFields.map(x => [x, () => Eval('', [sel.ref, x], sel2.model.getType(x)!)])), + }).execute() + const upsert = rawUpsert.map(row => ({ + ...mapValues(update, (_, key) => getCell(row, key)), + ...Object.fromEntries(primaryFields.map(x => [x, getCell(row, x)])), + })) + return this.database.upsert(table, upsert) + } else { + const data = await this.database.get(table as never, query) + for (const row of data) { + this._update(sel, primaryFields, updateFields, update, row) + } + return { matched: data.length } + } + } + + _create(table: string, data: {}) { + const model = this.model(table) + data = this.sql.dump(data, model) + const keys = Object.keys(data) + const sql = `INSERT INTO ${escapeId(table)} (${this._joinKeys(keys)}) VALUES (${Array(keys.length).fill('?').join(', ')})` + return this._run(sql, keys.map(key => data[key] ?? null), () => this._get(`SELECT last_insert_rowid() AS id`)) + } + + async create(sel: Selection.Mutable, data: {}) { + const { model, table } = sel + const { id } = this._create(table, data) + const { autoInc, primary } = model + if (!autoInc || Array.isArray(primary)) return data as any + return { ...data, [primary]: id } + } + + async upsert(sel: Selection.Mutable, data: any[], keys: string[]) { + if (!data.length) return {} + const { model, table, ref } = sel + const fields = model.avaiableFields() + const result = { inserted: 0, matched: 0, modified: 0 } + const dataFields = [...new Set(Object.keys(Object.assign({}, ...data)).map((key) => { + return Object.keys(fields).find(field => field === key || key.startsWith(field + '.'))! + }))] + let updateFields = difference(dataFields, keys) + if (!updateFields.length) updateFields = [dataFields[0]] + // Error: Expression tree is too large (maximum depth 1000) + const step = Math.floor(960 / keys.length) + for (let i = 0; i < data.length; i += step) { + const chunk = data.slice(i, i + step) + const results = await this.database.get(table as never, { + $or: chunk.map(item => Object.fromEntries(keys.map(key => [key, item[key]]))), + }) + for (const item of chunk) { + const row = results.find(row => { + // flatten key to respect model + row = model.format(row) + return keys.every(key => deepEqual(row[key], item[key], true)) + }) + if (row) { + this._update(sel, keys, updateFields, item, row) + result.matched++ + } else { + this._create(table, executeUpdate(model.create(), item, ref)) + result.inserted++ + } + } + } + return result + } + + async withTransaction(callback: () => Promise) { + if (this._transactionTask) await this._transactionTask.catch(() => { }) + return this._transactionTask = new Promise((resolve, reject) => { + this._run('BEGIN TRANSACTION') + callback().then( + () => resolve(this._run('COMMIT')), + (e) => (this._run('ROLLBACK'), reject(e)), + ) + }) + } +} + +export namespace SQLiteDriver { + export interface Config { + path: string + } + + export const Config: z = z.object({ + path: z.string().role('path').required(), + }).i18n({ + 'en-US': enUS, + 'zh-CN': zhCN, + }) +} + +export default SQLiteDriver diff --git a/packages/node-sqlite/src/locales/en-US.yml b/packages/node-sqlite/src/locales/en-US.yml new file mode 100644 index 00000000..4728d564 --- /dev/null +++ b/packages/node-sqlite/src/locales/en-US.yml @@ -0,0 +1 @@ +path: Database path. diff --git a/packages/node-sqlite/src/locales/zh-CN.yml b/packages/node-sqlite/src/locales/zh-CN.yml new file mode 100644 index 00000000..b09c9d6b --- /dev/null +++ b/packages/node-sqlite/src/locales/zh-CN.yml @@ -0,0 +1 @@ +path: 数据库路径。 diff --git a/packages/node-sqlite/src/node-sqlite.d.ts b/packages/node-sqlite/src/node-sqlite.d.ts new file mode 100644 index 00000000..82e54b7d --- /dev/null +++ b/packages/node-sqlite/src/node-sqlite.d.ts @@ -0,0 +1,42 @@ +declare module 'node:sqlite' { + // Type conversion between JavaScript and SQLite data types + export type SQLiteDataTypes = { + NULL: null + INTEGER: number | BigInt + REAL: number + TEXT: string + BLOB: Uint8Array + } + + // Class representing a SQLite Database connection + export class DatabaseSync { + constructor(location: string, options?: DatabaseOptions) + + close(): void // Closes the database connection + exec(sql: string): void // Executes one or more SQL statements + open(): void // Opens the database + prepare(sql: string): StatementSync // Compiles SQL into a prepared statement + } + + // Options for DatabaseSync + export interface DatabaseOptions { + open?: boolean // Default is true, open on constructor + } + + // Class representing a prepared statement + export class StatementSync { + all(namedParameters?: Record, ...anonymousParameters: any[]): Record[] + expandedSQL(): string // Returns SQL with parameters replaced + get(namedParameters?: Record, ...anonymousParameters: any[]): Record | undefined + run(namedParameters?: Record, ...anonymousParameters: any[]): RunResult + setAllowBareNamedParameters(enabled: boolean): void // Enable/disable bare named parameters + setReadBigInts(enabled: boolean): void // Enable/disable reading INTEGER as BigInt + sourceSQL(): string // Returns the source SQL of the prepared statement + } + + // Result returned from running a prepared statement + export interface RunResult { + changes: number | BigInt // Rows modified, inserted, or deleted + lastInsertRowid: number | BigInt // Last inserted row ID + } +} diff --git a/packages/node-sqlite/tests/index.spec.ts b/packages/node-sqlite/tests/index.spec.ts new file mode 100644 index 00000000..da1a21e2 --- /dev/null +++ b/packages/node-sqlite/tests/index.spec.ts @@ -0,0 +1,32 @@ +import { join } from 'path' +import { Database } from 'minato' +import SQLiteDriver from '@minatojs/driver-node-sqlite' +import Logger from 'reggol' +import test from '@minatojs/tests' + +const logger = new Logger('sqlite') + +describe('@minatojs/driver-node-sqlite', () => { + const database = new Database() + + before(async () => { + logger.level = 2 + await database.connect(SQLiteDriver, { + path: join(__dirname, 'test.db'), + }) + }) + + after(async () => { + await database.dropAll() + await database.stopAll() + logger.level = 2 + }) + + test(database, { + query: { + list: { + elementQuery: false, + }, + }, + }) +}) diff --git a/packages/node-sqlite/tsconfig.json b/packages/node-sqlite/tsconfig.json new file mode 100644 index 00000000..74ac2c8d --- /dev/null +++ b/packages/node-sqlite/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.base", + "compilerOptions": { + "outDir": "lib", + "rootDir": "src", + }, + "include": [ + "src", + ], +} \ No newline at end of file diff --git a/yakumo.yml b/yakumo.yml index c4fed37a..167b2b9c 100644 --- a/yakumo.yml +++ b/yakumo.yml @@ -1,4 +1,5 @@ -- name: yakumo +- id: n0zkge + name: yakumo config: pipeline: build: @@ -6,6 +7,11 @@ - esbuild clean: - tsc --clean -- name: yakumo-esbuild -- name: yakumo-mocha -- name: yakumo-tsc +- id: b26v82 + name: yakumo-esbuild +- id: kth7fe + name: yakumo-mocha +- id: i8tmd8 + name: yakumo-tsc +- id: dra0z1 + name: yakumo/run From 263a19a6cf0a196c1599ad831c996805529123c2 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 15 Aug 2024 11:47:56 +0800 Subject: [PATCH 2/5] chore(node-sqlite): merge master features, update ci --- .github/workflows/test.yaml | 27 +++++++++++++ packages/node-sqlite/package.json | 4 +- packages/node-sqlite/src/index.ts | 48 +++++++++++++++++++++++- packages/node-sqlite/tests/index.spec.ts | 2 +- 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8fbd31a8..f1123f59 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -162,6 +162,33 @@ jobs: files: ./coverage/coverage-final.json name: codecov + node-sqlite: + name: node-sqlite (${{ matrix.node-version }}) + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + node-version: [22] + + steps: + - name: Check out + uses: actions/checkout@v4 + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - name: Install + run: yarn --no-immutable + - name: Unit Test + run: yarn test:json --experimental-sqlite node-sqlite + - name: Report Coverage + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage/coverage-final.json + name: codecov + test: name: ${{ matrix.driver-name }} (${{ matrix.node-version }}) runs-on: ubuntu-latest diff --git a/packages/node-sqlite/package.json b/packages/node-sqlite/package.json index d641781e..adb92977 100644 --- a/packages/node-sqlite/package.json +++ b/packages/node-sqlite/package.json @@ -24,12 +24,12 @@ "repository": { "type": "git", "url": "git+https://github.com/shigma/minato.git", - "directory": "packages/sqlite" + "directory": "packages/node-sqlite" }, "bugs": { "url": "https://github.com/shigma/minato/issues" }, - "homepage": "https://github.com/shigma/minato/packages/sqlite#readme", + "homepage": "https://github.com/shigma/minato/packages/node-sqlite#readme", "keywords": [ "orm", "database", diff --git a/packages/node-sqlite/src/index.ts b/packages/node-sqlite/src/index.ts index 519b5bf0..31c8044f 100644 --- a/packages/node-sqlite/src/index.ts +++ b/packages/node-sqlite/src/index.ts @@ -30,7 +30,7 @@ function getTypeDef({ deftype: type }: Field) { } } -export interface SQLiteFieldInfo { +interface SQLiteFieldInfo { cid: number name: string type: string @@ -39,6 +39,13 @@ export interface SQLiteFieldInfo { pk: boolean } +interface SQLiteMasterInfo { + type: string + name: string + tbl_name: string + sql: string +} + export class SQLiteDriver extends Driver { static name = 'sqlite' @@ -207,7 +214,7 @@ export class SQLiteDriver extends Driver { }) this.define({ - types: Field.number as any, + types: ['primary', ...Field.number as any], dump: value => value, load: value => isNullable(value) ? value : Number(value), }) @@ -412,6 +419,43 @@ export class SQLiteDriver extends Driver { ) }) } + + async getIndexes(table: string) { + const indexes = this._all(`SELECT type,name,tbl_name,sql FROM sqlite_master WHERE type = 'index' AND tbl_name = ?`, [table]) as SQLiteMasterInfo[] + const result: Driver.Index[] = [] + for (const { name, sql } of indexes) { + result.push({ + name, + unique: !sql || sql.toUpperCase().startsWith('CREATE UNIQUE'), + keys: this._parseIndexDef(sql), + }) + } + return result + } + + async createIndex(table: string, index: Driver.Index) { + const name = index.name ?? Object.entries(index.keys).map(([key, direction]) => `${key}_${direction ?? 'asc'}`).join('+') + const keyFields = Object.entries(index.keys).map(([key, direction]) => `${escapeId(key)} ${direction ?? 'asc'}`).join(', ') + await this._run(`create ${index.unique ? 'UNIQUE' : ''} index ${escapeId(name)} ON ${escapeId(table)} (${keyFields})`) + } + + async dropIndex(table: string, name: string) { + await this._run(`DROP INDEX ${escapeId(name)}`) + } + + _parseIndexDef(def: string) { + if (!def) return {} + try { + const keys = {}, matches = def.match(/\((.*)\)/)! + matches[1].split(',').forEach((key) => { + const [name, direction] = key.trim().split(' ') + keys[name.startsWith('`') ? name.slice(1, -1) : name] = direction?.toLowerCase() === 'desc' ? 'desc' : 'asc' + }) + return keys + } catch { + return {} + } + } } export namespace SQLiteDriver { diff --git a/packages/node-sqlite/tests/index.spec.ts b/packages/node-sqlite/tests/index.spec.ts index da1a21e2..5245e6bb 100644 --- a/packages/node-sqlite/tests/index.spec.ts +++ b/packages/node-sqlite/tests/index.spec.ts @@ -10,7 +10,7 @@ describe('@minatojs/driver-node-sqlite', () => { const database = new Database() before(async () => { - logger.level = 2 + logger.level = 3 await database.connect(SQLiteDriver, { path: join(__dirname, 'test.db'), }) From d5f497aa4b8564e17c11f873c93ee6afc359faea Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 15 Aug 2024 12:14:32 +0800 Subject: [PATCH 3/5] chore: update --- packages/node-sqlite/package.json | 8 ++++---- packages/node-sqlite/readme.md | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/node-sqlite/package.json b/packages/node-sqlite/package.json index adb92977..fda701b3 100644 --- a/packages/node-sqlite/package.json +++ b/packages/node-sqlite/package.json @@ -1,6 +1,6 @@ { "name": "@minatojs/driver-node-sqlite", - "version": "4.4.1", + "version": "4.5.0", "description": "node:sqlite Driver for Minato", "type": "module", "main": "lib/index.cjs", @@ -23,13 +23,13 @@ "license": "MIT", "repository": { "type": "git", - "url": "git+https://github.com/shigma/minato.git", + "url": "git+https://github.com/cordiverse/minato.git", "directory": "packages/node-sqlite" }, "bugs": { - "url": "https://github.com/shigma/minato/issues" + "url": "https://github.com/cordiverse/minato/issues" }, - "homepage": "https://github.com/shigma/minato/packages/node-sqlite#readme", + "homepage": "https://github.com/cordiverse/minato/packages/node-sqlite#readme", "keywords": [ "orm", "database", diff --git a/packages/node-sqlite/readme.md b/packages/node-sqlite/readme.md index d40786df..4f551404 100644 --- a/packages/node-sqlite/readme.md +++ b/packages/node-sqlite/readme.md @@ -1,3 +1,7 @@ # @minatojs/driver-node-sqlite +[![downloads](https://img.shields.io/npm/dm/@minatojs/driver-node-sqlite?style=flat-square)](https://www.npmjs.com/package/@minatojs/driver-node-sqlite) +[![npm](https://img.shields.io/npm/v/@minatojs/driver-node-sqlite?style=flat-square)](https://www.npmjs.com/package/@minatojs/driver-node-sqlite) +[![GitHub](https://img.shields.io/github/license/cordiverse/minato?style=flat-square)](https://github.com/cordiverse/minato/blob/master/LICENSE) + node:sqlite Driver for Minato. From 62a484e5102d42855f19caee6e8bf759dc10561a Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 15 Aug 2024 12:51:23 +0800 Subject: [PATCH 4/5] fix(tests): migrate chai v5, lint --- packages/tests/src/setup.ts | 44 ++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/tests/src/setup.ts b/packages/tests/src/setup.ts index e80e20a8..b5395956 100644 --- a/packages/tests/src/setup.ts +++ b/packages/tests/src/setup.ts @@ -1,4 +1,4 @@ -import chai, { use } from 'chai' +import { config, use, util } from 'chai' import promised from 'chai-as-promised' import shape from './shape' import { isNullable } from 'cosmokit' @@ -8,44 +8,44 @@ use(promised) function type(obj) { if (typeof obj === 'undefined') { - return 'undefined'; + return 'undefined' } if (obj === null) { - return 'null'; + return 'null' } - const stringTag = obj[Symbol.toStringTag]; + const stringTag = obj[Symbol.toStringTag] if (typeof stringTag === 'string') { - return stringTag; + return stringTag } - const sliceStart = 8; - const sliceEnd = -1; - return Object.prototype.toString.call(obj).slice(sliceStart, sliceEnd); + const sliceStart = 8 + const sliceEnd = -1 + return Object.prototype.toString.call(obj).slice(sliceStart, sliceEnd) } function getEnumerableKeys(target) { - var keys: string[] = []; - for (var key in target) { - keys.push(key); + const keys: string[] = [] + for (const key in target) { + keys.push(key) } - return keys; + return keys } function getEnumerableSymbols(target) { - var keys: symbol[] = []; - var allKeys = Object.getOwnPropertySymbols(target); - for (var i = 0; i < allKeys.length; i += 1) { - var key = allKeys[i]; + const keys: symbol[] = [] + const allKeys = Object.getOwnPropertySymbols(target) + for (let i = 0; i < allKeys.length; i += 1) { + const key = allKeys[i] if (Object.getOwnPropertyDescriptor(target, key)?.enumerable) { - keys.push(key); + keys.push(key) } } - return keys; + return keys } -chai.config.deepEqual = (expected, actual, options) => { - return chai.util.eql(expected, actual, { +config.deepEqual = (expected, actual, options) => { + return util.eql(expected, actual, { comparator: (expected, actual) => { if (isNullable(expected) && isNullable(actual)) return true if (type(expected) === 'Object' && type(actual) === 'Object') { @@ -55,9 +55,9 @@ chai.config.deepEqual = (expected, actual, options) => { ...getEnumerableSymbols(expected), ...getEnumerableSymbols(actual), ]) - return [...keys].every(key => chai.config.deepEqual!(expected[key], actual[key], options)) + return [...keys].every(key => config.deepEqual!(expected[key], actual[key], options)) } return null - } + }, }) } From f9e5745258c3aa366f727ee698804d1b4385b823 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Fri, 16 Aug 2024 11:39:46 +0800 Subject: [PATCH 5/5] chore: revert: yakumo.yml --- packages/node-sqlite/.npmignore | 2 -- yakumo.yml | 14 ++++---------- 2 files changed, 4 insertions(+), 12 deletions(-) delete mode 100644 packages/node-sqlite/.npmignore diff --git a/packages/node-sqlite/.npmignore b/packages/node-sqlite/.npmignore deleted file mode 100644 index 7e5fcbc1..00000000 --- a/packages/node-sqlite/.npmignore +++ /dev/null @@ -1,2 +0,0 @@ -.DS_Store -tsconfig.tsbuildinfo diff --git a/yakumo.yml b/yakumo.yml index 167b2b9c..c4fed37a 100644 --- a/yakumo.yml +++ b/yakumo.yml @@ -1,5 +1,4 @@ -- id: n0zkge - name: yakumo +- name: yakumo config: pipeline: build: @@ -7,11 +6,6 @@ - esbuild clean: - tsc --clean -- id: b26v82 - name: yakumo-esbuild -- id: kth7fe - name: yakumo-mocha -- id: i8tmd8 - name: yakumo-tsc -- id: dra0z1 - name: yakumo/run +- name: yakumo-esbuild +- name: yakumo-mocha +- name: yakumo-tsc