feat: use PostgreSQL

Author: tianheg

.github/workflows/json2db.yml

@@ -0,0 +1,24 @@
+name: json2db
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: oven-sh/setup-bun@v1
+
+      - name: Build
+        run: |
+          bun install
+          bun run scripts/jsonToDb.js
+        env:
+          POSTGRES_URL: ${{ secrets.POSTGRES_URL }}

README.md

@@ -18,6 +18,8 @@
 
 ## Problems
 
+### jsonToDb 数据太多上传超时
+
 ### Swagger UI cannot display endpoints(SOLVED)
 
 > The bug likely occurs due to the asynchronous nature of the `import()` function used in the `app.register()` calls. When you use `import()` inside `app.register()`, it returns a promise that resolves to the module, and Fastify's `register` method might not be set up to handle promises directly in this manner. This can lead to a race condition where Fastify starts setting up routes before the Swagger plugins are fully registered and configured, resulting in the Swagger UI not being aware of any routes.

bun.lockb

@@ -4,22 +4,25 @@
   "type": "module",
   "devDependencies": {
     "@biomejs/biome": "^1.8.2",
+    "cheerio": "^1.0.0-rc.12",
+    "dotenv": "^16.4.5",
     "nodemon": "^3.1.4",
+    "pino": "^9.2.0",
     "pino-pretty": "^11.2.1",
-    "tap": "^19.2.5",
-    "cheerio": "^1.0.0-rc.12",
-    "pino": "^9.2.0"
+    "tap": "^19.2.5"
   },
   "dependencies": {
     "@fastify/autoload": "^5.10.0",
     "@fastify/compress": "^7.0.3",
     "@fastify/cors": "^9.0.1",
     "@fastify/helmet": "^11.1.1",
+    "@fastify/postgres": "^5.2.2",
     "@fastify/rate-limit": "^9.1.0",
     "@fastify/swagger": "^8.14.0",
     "@fastify/swagger-ui": "^3.1.0",
     "@fastify/under-pressure": "^8.5.0",
-    "fastify": "^4.28.0"
+    "fastify": "^4.28.0",
+    "pg": "^8.12.0"
   },
   "scripts": {
     "dev": "export NODE_ENV=development && nodemon .",

package.json

@@ -31,6 +31,19 @@ export default async function registerRoutes(app) {
     });
   });
 
+  app.get("/calc", (request, reply) => {
+    const client = app.pg.connect();
+
+    console.log(client);
+    // const sumResult = client.query < { sum } > "SELECT 2 + 2 as sum";
+
+    // client.release();
+
+    // return {
+    //   sum: sumResult.rows,
+    // };
+  });
+
   createRoute(app, "/books", booksData, { schema: paginationSchema });
   createRoute(app, "/feeds", feedsData, { schema: paginationSchema });
   createRoute(app, "/movies", moviesData, { schema: paginationSchema });

routes.js

@@ -0,0 +1,72 @@
+import * as dotenv from "dotenv";
+dotenv.config();
+
+import { Client } from "pg";
+
+// Set up the PostgreSQL client with your configuration
+const client = new Client({
+  connectionString: process.env.POSTGRES_URL,
+});
+
+async function createTableIfNotExist(client, tableName) {
+  const createTableQuery = `
+    CREATE TABLE IF NOT EXISTS ${tableName} (
+      id SERIAL PRIMARY KEY,
+      json_data JSONB NOT NULL
+    );
+  `;
+  await client.query(createTableQuery);
+  console.log(`Table ${tableName} is ready, or already exists.`);
+}
+
+async function truncateTable(client, tableName) {
+  const truncateQuery = `TRUNCATE TABLE ${tableName};`;
+  await client.query(truncateQuery);
+  console.log(`Truncated table ${tableName}`);
+}
+
+async function uploadJsonDataFromFile(client, relativePath, tableName) {
+  try {
+    const module = await import(`../${relativePath}`);
+    const jsonDataFromFile = module.default;
+
+    await createTableIfNotExist(client, tableName);
+    await truncateTable(client, tableName);
+
+    // Assuming jsonDataFromFile is an array of objects
+    for (const data of jsonDataFromFile) {
+      const query = {
+        text: `INSERT INTO ${tableName}(json_data) VALUES(\$1)`,
+        values: [JSON.stringify(data)], // Convert data to JSON string
+      };
+      await client.query(query);
+    }
+
+    console.log(`Data from ${relativePath} inserted successfully into ${tableName}`);
+  } catch (err) {
+    console.error(`Error uploading data from file ${relativePath}:`, err);
+  }
+}
+
+async function uploadAllFiles(client) {
+  const files = ["books.js", "feeds.js", "movies.js", "music.js", "musicals.js", "prompts.js", "series.js", "words.js"];
+
+  for (const file of files) {
+    const tableName = file.replace('.js', ''); // Remove '.js' to form the table name
+    await uploadJsonDataFromFile(client, `data/${file}`, tableName);
+  }
+}
+
+async function main() {
+  try {
+    await client.connect();
+    await uploadAllFiles(client);
+    console.log("Data uploaded successfully");
+  } catch (err) {
+    console.error("Error uploading data:", err);
+  } finally {
+    await client.end();
+    console.log("Disconnected from the database.");
+  }
+}
+main();

scripts/jsonToDb.js

@@ -4,10 +4,14 @@ import helmet from "@fastify/helmet";
 import rateLimit from "@fastify/rate-limit";
 import swagger from "@fastify/swagger";
 import swaggerUI from "@fastify/swagger-ui";
+import postgres from "@fastify/postgres";
 import Fastify from "fastify";
 import pino from "pino";
 import pretty from "pino-pretty";
 
+import * as dotenv from "dotenv";
+dotenv.config();
+
 import registerRoutes from "./routes.js";
 
 // logger
@@ -62,6 +66,11 @@ await app.register(swaggerUI, {
   },
 });
 
+// database
+await app.register(postgres, {
+  connectionString: process.env.POSTGRES_URL,
+});
+
 /// routes
 registerRoutes(app);