From faa5eb25ef7e2dbdddf34294c204f37fa8db1957 Mon Sep 17 00:00:00 2001 From: shinshin86 Date: Mon, 7 Oct 2024 08:14:19 +0900 Subject: [PATCH] add csv export and import --- README.md | 38 ++++++ e2e/csv-export-and-import.test.ts | 208 ++++++++++++++++++++++++++++++ src/neverchange.ts | 60 +++++++++ src/parser.ts | 30 +++++ 4 files changed, 336 insertions(+) create mode 100644 e2e/csv-export-and-import.test.ts create mode 100644 src/parser.ts diff --git a/README.md b/README.md index d30315b..ddf12a7 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,44 @@ const blobData = convertToUint8Array(row.blobColumn); - **Complex Data Types**: While NeverChangeDB handles most SQLite data types seamlessly, complex types like JSON or custom data structures may require additional processing when dumping or importing. - **Cross-Browser Compatibility**: Although the core functionality is designed to work across modern browsers, some advanced features or performance optimizations may vary between different browser environments. Always test thoroughly in your target browsers. +#### CSV Export and Import + +NeverChangeDB also supports CSV export and import functionality, allowing you to easily work with CSV files in your database. + +##### CSV Export + +You can export a table to a CSV format using the `dumpTableToCSV` method: + +```typescript +const db = new NeverChangeDB('myDatabase'); +await db.init(); + +/* We will assume that you have added tables and information */ + +const csvContent = await db.dumpTableToCSV('your_table'); +console.log('CSV Export:', csvContent); + +await db.close(); +``` + +This will export the contents of `your_table` to a CSV string. + +##### CSV Import + +You can import CSV content into a table using the `importCSVToTable` method: + +```typescript +const db = new NeverChangeDB('myDatabase'); +await db.init(); + +const csvContent = `id,name,email\n1,John Doe,john@example.com\n2,Jane Smith,jane@example.com`; +await db.importCSVToTable('your_table', csvContent); + +await db.close(); +``` + +This will insert the CSV data into the `your_table` table. Ensure the table is created beforehand and the columns match the CSV headers. + ## For Developers ### Setup diff --git a/e2e/csv-export-and-import.test.ts b/e2e/csv-export-and-import.test.ts new file mode 100644 index 0000000..530b923 --- /dev/null +++ b/e2e/csv-export-and-import.test.ts @@ -0,0 +1,208 @@ +import { test, expect } from "@playwright/test"; + +// sleep function for debugging +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test.describe("NeverChangeDB CSV Export and Import", () => { + test.beforeEach(async ({ page }) => { + await page.goto("/"); + await sleep(100); + }); + + test("should export table to CSV correctly", async ({ page }) => { + // Initialize the database + const csvContent = await page.evaluate(async () => { + const db = new (window as any).NeverChangeDB("csv-export-db"); + await db.init(); + await db.execute(` + CREATE TABLE test_table ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + email TEXT + ) + `); + await db.execute("INSERT INTO test_table (name, email) VALUES (?, ?)", [ + "John Doe", + "john@example.com", + ]); + await db.execute("INSERT INTO test_table (name, email) VALUES (?, ?)", [ + "Jane Smith", + "jane@example.com", + ]); + + // Export table to CSV + return await db.dumpTableToCSV("test_table"); + }); + + expect(csvContent).toContain("id,name,email"); + expect(csvContent).toContain("John Doe,john@example.com"); + expect(csvContent).toContain("Jane Smith,jane@example.com"); + }); + + test("should import CSV content into table correctly", async ({ page }) => { + const csvContent = + "id,name,email\n1,John Doe,john@example.com\n2,Jane Smith,jane@example.com"; + + // Import CSV into a new database + const importedData = await page.evaluate(async (csvContent) => { + const db = new (window as any).NeverChangeDB("csv-import-db"); + await db.init(); + await db.execute(` + CREATE TABLE test_table ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + email TEXT + ) + `); + + // Import the CSV content into the table + await db.importCSVToTable("test_table", csvContent); + + return await db.query("SELECT * FROM test_table"); + }, csvContent); + + expect(importedData).toHaveLength(2); + expect(importedData[0].name).toBe("John Doe"); + expect(importedData[0].email).toBe("john@example.com"); + expect(importedData[1].name).toBe("Jane Smith"); + expect(importedData[1].email).toBe("jane@example.com"); + }); + + test("should handle empty CSV export and import", async ({ page }) => { + const csvContent = await page.evaluate(async () => { + const db = new (window as any).NeverChangeDB("empty-csv-db"); + await db.init(); + await db.execute(` + CREATE TABLE test_table ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + email TEXT + ) + `); + + // Export empty table to CSV + return await db.dumpTableToCSV("test_table"); + }); + + expect(csvContent).toContain("id,name,email"); + expect(csvContent.split("\r\n").filter(Boolean)).toHaveLength(1); // Only headers + + // Import empty CSV content back into a new table + const importedData = await page.evaluate(async (csvContent) => { + const db = new (window as any).NeverChangeDB("empty-csv-import-db"); + await db.init(); + await db.execute(` + CREATE TABLE test_table ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + email TEXT + ) + `); + + await db.importCSVToTable("test_table", csvContent); + + return await db.query("SELECT * FROM test_table"); + }, csvContent); + + expect(importedData).toHaveLength(0); + }); + + test("should handle CSV with special characters (default behavior)", async ({ + page, + }) => { + const csvContent = await page.evaluate(async () => { + const db = new (window as any).NeverChangeDB("csv-special-chars-db"); + await db.init(); + await db.execute(` + CREATE TABLE test_table ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + description TEXT + ) + `); + await db.execute( + "INSERT INTO test_table (name, description) VALUES (?, ?)", + ["John Doe", 'This is a test with, commas and "quotes".'], + ); + + return await db.dumpTableToCSV("test_table"); + }); + + expect(csvContent).toBe( + `id,name,description\r\n1,John Doe,"This is a test with, commas and ""quotes""."\r\n`, + ); + + // Import the CSV content back into the table and validate + const importedData = await page.evaluate(async (csvContent) => { + const db = new (window as any).NeverChangeDB("csv-special-import-db"); + await db.init(); + await db.execute(` + CREATE TABLE test_table ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + description TEXT + ) + `); + + await db.importCSVToTable("test_table", csvContent); + + return await db.query("SELECT * FROM test_table"); + }, csvContent); + + expect(importedData).toHaveLength(1); + expect(importedData[0].name).toBe("John Doe"); + expect(importedData[0].description).toBe( + 'This is a test with, commas and "quotes".', + ); + }); + + test("should handle CSV with special characters (quoteAllFields option)", async ({ + page, + }) => { + const csvContent = await page.evaluate(async () => { + const db = new (window as any).NeverChangeDB("csv-quote-all-fields-db"); + await db.init(); + await db.execute(` + CREATE TABLE test_table ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + description TEXT + ) + `); + await db.execute( + "INSERT INTO test_table (name, description) VALUES (?, ?)", + ["John Doe", 'This is a test with, commas and "quotes".'], + ); + + // Export table to CSV with quoteAllFields option + return await db.dumpTableToCSV("test_table", { quoteAllFields: true }); + }); + + expect(csvContent).toBe( + `"id","name","description"\r\n"1","John Doe","This is a test with, commas and ""quotes""."\r\n`, + ); + + // Import the CSV content back into the table and validate + const importedData = await page.evaluate(async (csvContent) => { + const db = new (window as any).NeverChangeDB("csv-special-import-db"); + await db.init(); + await db.execute(` + CREATE TABLE test_table ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + description TEXT + ) + `); + + await db.importCSVToTable("test_table", csvContent); + + return await db.query("SELECT * FROM test_table"); + }, csvContent); + + expect(importedData).toHaveLength(1); + expect(importedData[0].name).toBe("John Doe"); + expect(importedData[0].description).toBe( + 'This is a test with, commas and "quotes".', + ); + }); +}); diff --git a/src/neverchange.ts b/src/neverchange.ts index 1ebe0ce..2ec4df7 100644 --- a/src/neverchange.ts +++ b/src/neverchange.ts @@ -7,6 +7,7 @@ import { Migration, } from "./types"; import { initialMigration } from "./migrations"; +import { parseCSVLine } from "./parser"; export class NeverChangeDB implements INeverChangeDB { private dbPromise: Promise< @@ -315,4 +316,63 @@ export class NeverChangeDB implements INeverChangeDB { throw error; } } + + async dumpTableToCSV( + tableName: string, + options: { quoteAllFields?: boolean } = {}, + ): Promise { + const rows = await this.query(`SELECT * FROM ${tableName}`); + + const columnNames = Object.keys( + rows[0] || + (await this.query(`PRAGMA table_info(${tableName})`)).reduce( + (acc: any, col: any) => ({ ...acc, [col.name]: "" }), + {}, + ), + ) + .map((col) => (options.quoteAllFields ? `"${col}"` : col)) + .join(","); + + if (rows.length === 0) { + return `${columnNames}\r\n`; + } + + const escapeCSVField = (field: any): string => { + const strValue = field?.toString() || ""; + const needsQuoting = + options.quoteAllFields || + strValue.includes(",") || + strValue.includes("\n") || + strValue.includes('"'); + + if (needsQuoting) { + return `"${strValue.replace(/"/g, '""')}"`; + } + + return strValue; + }; + + const csvRows = rows.map((row) => + Object.values(row).map(escapeCSVField).join(","), + ); + + return `${columnNames}\r\n${csvRows.join("\r\n")}\r\n`; + } + + async importCSVToTable(tableName: string, csvContent: string): Promise { + const [headerLine, ...dataLines] = csvContent + .split(/\r?\n/) + .filter(Boolean); + const columns = headerLine.split(","); + + for (const line of dataLines) { + const values = parseCSVLine(line); + const placeholders = columns.map(() => "?").join(","); + + await this.execute( + `INSERT INTO ${tableName} (${columns.join(",")}) VALUES (${placeholders})`, + values, + ); + } + } } diff --git a/src/parser.ts b/src/parser.ts new file mode 100644 index 0000000..caf0145 --- /dev/null +++ b/src/parser.ts @@ -0,0 +1,30 @@ +export const parseCSVLine = (line: string): string[] => { + const result: string[] = []; + let currentField = ""; + let insideQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + + if (char === '"') { + if (insideQuotes && line[i + 1] === '"') { + currentField += '"'; + i++; // Skip the next quote + } else { + // Toggle the start or end of a quoted field + insideQuotes = !insideQuotes; + } + } else if (char === "," && !insideQuotes) { + // Treat comma as a field separator + result.push(currentField); + currentField = ""; + } else { + // Add the character as part of the current field + currentField += char; + } + } + + // Add the last field to the result + result.push(currentField); + return result; +};