From 62aae0d244f5216bc105b397f7b75c83e7172cb9 Mon Sep 17 00:00:00 2001 From: Innei Date: Tue, 23 Jul 2024 22:20:41 +0800 Subject: [PATCH] feat: db cleaner (#151) * feat: db cleaner Signed-off-by: Innei * chore: add test case Signed-off-by: Innei * update Signed-off-by: Innei * refactor: data struct Signed-off-by: Innei * feat: cleaner old data Signed-off-by: Innei * update Signed-off-by: Innei --------- Signed-off-by: Innei --- package.json | 2 + pnpm-lock.yaml | 50 ++++++ setup-file.ts | 1 + src/renderer/src/database/db.ts | 38 +++-- src/renderer/src/database/db_schema.ts | 16 +- src/renderer/src/database/index.ts | 1 - src/renderer/src/database/model.ts | 31 ---- .../src/database/models/entry-related.ts | 10 -- src/renderer/src/database/models/entry.ts | 10 -- .../src/database/models/feed-entry.ts | 10 -- src/renderer/src/database/models/feed.ts | 10 -- src/renderer/src/database/models/index.ts | 5 - .../src/database/models/subscription.ts | 10 -- src/renderer/src/database/models/unread.ts | 10 -- src/renderer/src/database/models/user.ts | 3 - src/renderer/src/database/schemas/cleaner.ts | 5 + src/renderer/src/database/schemas/entry.ts | 8 + src/renderer/src/database/schemas/feed.ts | 11 +- .../src/database/schemas/subscription.ts | 5 + src/renderer/src/database/types.ts | 5 - src/renderer/src/initialize/hydrate.ts | 2 +- src/renderer/src/initialize/index.ts | 2 + src/renderer/src/models/types.ts | 3 + src/renderer/src/providers/user-provider.tsx | 3 + .../__snapshots__/cleaner.spec.ts.snap | 85 ++++++++++ src/renderer/src/services/base.ts | 8 - src/renderer/src/services/cleaner.spec.ts | 157 ++++++++++++++++++ src/renderer/src/services/cleaner.ts | 90 ++++++++++ src/renderer/src/services/entry-related.ts | 12 +- src/renderer/src/services/entry.ts | 57 ++++++- src/renderer/src/services/feed-entry.ts | 20 --- src/renderer/src/services/feed-unread.ts | 13 +- src/renderer/src/services/feed.ts | 12 +- src/renderer/src/services/index.ts | 1 - src/renderer/src/services/subscription.ts | 32 +++- src/renderer/src/store/entry/store.ts | 9 +- src/renderer/src/store/user/hooks.ts | 2 +- src/renderer/src/store/user/store.ts | 2 +- src/renderer/src/store/utils/clear.ts | 13 +- vitest.config.ts | 36 ++++ 40 files changed, 610 insertions(+), 190 deletions(-) create mode 100644 setup-file.ts delete mode 100644 src/renderer/src/database/model.ts delete mode 100644 src/renderer/src/database/models/entry-related.ts delete mode 100644 src/renderer/src/database/models/entry.ts delete mode 100644 src/renderer/src/database/models/feed-entry.ts delete mode 100644 src/renderer/src/database/models/feed.ts delete mode 100644 src/renderer/src/database/models/index.ts delete mode 100644 src/renderer/src/database/models/subscription.ts delete mode 100644 src/renderer/src/database/models/unread.ts delete mode 100644 src/renderer/src/database/models/user.ts create mode 100644 src/renderer/src/database/schemas/cleaner.ts create mode 100644 src/renderer/src/database/schemas/entry.ts create mode 100644 src/renderer/src/database/schemas/subscription.ts delete mode 100644 src/renderer/src/database/types.ts create mode 100644 src/renderer/src/services/__snapshots__/cleaner.spec.ts.snap create mode 100644 src/renderer/src/services/cleaner.spec.ts create mode 100644 src/renderer/src/services/cleaner.ts delete mode 100644 src/renderer/src/services/feed-entry.ts create mode 100644 vitest.config.ts diff --git a/package.json b/package.json index 4d8534fd36..25b0510064 100644 --- a/package.json +++ b/package.json @@ -146,6 +146,7 @@ "electron-vite": "^2.3.0", "eslint": "^9.7.0", "eslint-config-hyoban": "3.0.0-beta.27", + "fake-indexeddb": "6.0.0", "hono": "4.4.7", "lint-staged": "15.2.7", "postcss": "8.4.39", @@ -157,6 +158,7 @@ "tailwindcss": "3.4.6", "typescript": "^5.5.3", "vite": "^5.3.4", + "vite-tsconfig-paths": "4.3.2", "vitest": "2.0.3" }, "pnpm": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 12dda5a80c..a82f3796d5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -362,6 +362,9 @@ importers: eslint-config-hyoban: specifier: 3.0.0-beta.27 version: 3.0.0-beta.27(eslint@9.7.0)(tailwindcss@3.4.6)(typescript@5.5.3) + fake-indexeddb: + specifier: 6.0.0 + version: 6.0.0 hono: specifier: 4.4.7 version: 4.4.7(patch_hash=ycbk46disqruhfjducp47b5fl4) @@ -395,6 +398,9 @@ importers: vite: specifier: ^5.3.4 version: 5.3.4(@types/node@20.14.11) + vite-tsconfig-paths: + specifier: 4.3.2 + version: 4.3.2(typescript@5.5.3)(vite@5.3.4(@types/node@20.14.11)) vitest: specifier: 2.0.3 version: 2.0.3(@types/node@20.14.11) @@ -3979,6 +3985,10 @@ packages: resolution: {integrity: sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA==} engines: {'0': node >=0.6.0} + fake-indexeddb@6.0.0: + resolution: {integrity: sha512-YEboHE5VfopUclOck7LncgIqskAqnv4q0EWbYCaxKKjAvO93c+TJIaBuGy8CBFdbg9nKdpN3AuPRwVBJ4k7NrQ==} + engines: {node: '>=18'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -4295,6 +4305,9 @@ packages: resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} engines: {node: '>=10'} + globrex@0.1.2: + resolution: {integrity: sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==} + gopd@1.0.1: resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} @@ -6387,6 +6400,16 @@ packages: ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} + tsconfck@3.1.1: + resolution: {integrity: sha512-00eoI6WY57SvZEVjm13stEVE90VkEdJAFGgpFLTsZbJyW/LwFQ7uQxJHWpZ2hzSWgCPKc9AnBnNP+0X7o3hAmQ==} + engines: {node: ^18 || >=20} + hasBin: true + peerDependencies: + typescript: ^5.0.0 + peerDependenciesMeta: + typescript: + optional: true + tslib@2.6.3: resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==} @@ -6581,6 +6604,14 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} hasBin: true + vite-tsconfig-paths@4.3.2: + resolution: {integrity: sha512-0Vd/a6po6Q+86rPlntHye7F31zA2URZMbH8M3saAZ/xR9QoGN/L21bxEGfXdWmFdNkqPpRdxFT7nmNe12e9/uA==} + peerDependencies: + vite: '*' + peerDependenciesMeta: + vite: + optional: true + vite@5.3.4: resolution: {integrity: sha512-Cw+7zL3ZG9/NZBB8C+8QbQZmR54GwqIz+WMI4b3JgdYJvX+ny9AjJXqkGQlDXSXRP9rP0B4tbciRMOVEKulVOA==} engines: {node: ^18.0.0 || >=20.0.0} @@ -11193,6 +11224,8 @@ snapshots: extsprintf@1.4.1: optional: true + fake-indexeddb@6.0.0: {} + fast-deep-equal@3.1.3: {} fast-equals@5.0.1: {} @@ -11562,6 +11595,8 @@ snapshots: merge2: 1.4.1 slash: 3.0.0 + globrex@0.1.2: {} + gopd@1.0.1: dependencies: get-intrinsic: 1.2.4 @@ -13889,6 +13924,10 @@ snapshots: ts-interface-checker@0.1.13: {} + tsconfck@3.1.1(typescript@5.5.3): + optionalDependencies: + typescript: 5.5.3 + tslib@2.6.3: {} type-check@0.4.0: @@ -14098,6 +14137,17 @@ snapshots: - supports-color - terser + vite-tsconfig-paths@4.3.2(typescript@5.5.3)(vite@5.3.4(@types/node@20.14.11)): + dependencies: + debug: 4.3.5 + globrex: 0.1.2 + tsconfck: 3.1.1(typescript@5.5.3) + optionalDependencies: + vite: 5.3.4(@types/node@20.14.11) + transitivePeerDependencies: + - supports-color + - typescript + vite@5.3.4(@types/node@20.14.11): dependencies: esbuild: 0.21.5 diff --git a/setup-file.ts b/setup-file.ts new file mode 100644 index 0000000000..b054ed928d --- /dev/null +++ b/setup-file.ts @@ -0,0 +1 @@ +import "fake-indexeddb/auto" diff --git a/src/renderer/src/database/db.ts b/src/renderer/src/database/db.ts index f8ee999b30..1e73debb02 100644 --- a/src/renderer/src/database/db.ts +++ b/src/renderer/src/database/db.ts @@ -2,18 +2,25 @@ import type { Transaction } from "dexie" import Dexie from "dexie" import { LOCAL_DB_NAME } from "./constants" -import { dbSchemaV1, dbSchemaV2, dbSchemaV3 } from "./db_schema" -import type { DB_Base } from "./schemas/base" -import type { DB_FeedId } from "./schemas/feed" -import type { DBModel } from "./types" +import { + dbSchemaV1, + dbSchemaV2, + dbSchemaV3, + dbSchemaV4, + dbSchemaV5, +} from "./db_schema" +import type { DB_Cleaner } from "./schemas/cleaner" +import type { DB_Entry, DB_EntryRelated } from "./schemas/entry" +import type { DB_Feed, DB_FeedUnread } from "./schemas/feed" +import type { DB_Subscription } from "./schemas/subscription" export interface LocalDBSchemaMap { - entries: DB_Base - feeds: DB_Base - subscriptions: DB_Base - entryRelated: DB_Base - feedEntries: DB_FeedId - feedUnreads: DB_FeedId + entries: DB_Entry + feeds: DB_Feed + subscriptions: DB_Subscription + entryRelated: DB_EntryRelated + feedUnreads: DB_FeedUnread + cleaner: DB_Cleaner } // Define a local DB @@ -22,22 +29,23 @@ export class BrowserDB extends Dexie { public feeds: BrowserDBTable<"feeds"> public subscriptions: BrowserDBTable<"subscriptions"> public entryRelated: BrowserDBTable<"entryRelated"> - public feedEntries: BrowserDBTable<"feedEntries"> public feedUnreads: BrowserDBTable<"feedUnreads"> + public cleaner: BrowserDBTable<"cleaner"> constructor() { super(LOCAL_DB_NAME) this.version(1).stores(dbSchemaV1) - this.version(2).stores(dbSchemaV2) - .upgrade(this.upgradeToV2) + this.version(2).stores(dbSchemaV2).upgrade(this.upgradeToV2) this.version(3).stores(dbSchemaV3) + this.version(4).stores(dbSchemaV4) + this.version(5).stores(dbSchemaV5) this.entries = this.table("entries") this.feeds = this.table("feeds") this.subscriptions = this.table("subscriptions") this.entryRelated = this.table("entryRelated") - this.feedEntries = this.table("feedEntries") this.feedUnreads = this.table("feedUnreads") + this.cleaner = this.table("cleaner") } async upgradeToV2(trans: Transaction) { @@ -58,7 +66,7 @@ export const browserDB = new BrowserDB() export type BrowserDBSchema = { [t in keyof LocalDBSchemaMap]: { model: LocalDBSchemaMap[t] - table: Dexie.Table, string> + table: Dexie.Table }; } type BrowserDBTable = diff --git a/src/renderer/src/database/db_schema.ts b/src/renderer/src/database/db_schema.ts index b9f653d0ce..c86fb3d8db 100644 --- a/src/renderer/src/database/db_schema.ts +++ b/src/renderer/src/database/db_schema.ts @@ -3,7 +3,6 @@ export const dbSchemaV1 = { feeds: "&id", subscriptions: "&id", entryRelated: "&id", - feedEntries: "&feedId", feedUnreads: "&id", } @@ -13,6 +12,19 @@ export const dbSchemaV2 = { } export const dbSchemaV3 = { - ...dbSchemaV1, + ...dbSchemaV2, + feedEntries: null, + subscriptions: "&id, userId, &feedId", } + +export const dbSchemaV4 = { + ...dbSchemaV3, + entries: "&id, feedId", + subscriptions: "&id, userId, feedId", +} + +export const dbSchemaV5 = { + ...dbSchemaV4, + cleaner: "&refId, visitedAt", +} diff --git a/src/renderer/src/database/index.ts b/src/renderer/src/database/index.ts index 3186a29980..3ac46bc8f6 100644 --- a/src/renderer/src/database/index.ts +++ b/src/renderer/src/database/index.ts @@ -1,7 +1,6 @@ import { browserDB } from "./db" export * from "./db" -export * from "./models" export * from "./schemas" export const DB_NOT_READY_OR_DISABLED = "Database is not ready or disabled" diff --git a/src/renderer/src/database/model.ts b/src/renderer/src/database/model.ts deleted file mode 100644 index 795f53a26d..0000000000 --- a/src/renderer/src/database/model.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * @see https://github.com/lobehub/lobe-chat/blob/adebf0a92167faad48581b0b0780cf8faeba362f/src/database/client/core/model.ts - */ - -import type Dexie from "dexie" -import type { ZodObject } from "zod" - -import type { BrowserDB, BrowserDBSchema } from "./db" -import { browserDB } from "./db" - -export class BaseModel< - N extends keyof BrowserDBSchema = any, - // T extends { id: string } = any, - // T = BrowserDBSchema[N]["table"], -> { - protected readonly db: BrowserDB - // used to data validation, but use now - - private readonly schema: ZodObject - private readonly _tableName: keyof BrowserDBSchema - - constructor(table: N, schema: ZodObject, db = browserDB) { - this.db = db - this.schema = schema - this._tableName = table - } - - get table() { - return this.db[this._tableName] as Dexie.Table - } -} diff --git a/src/renderer/src/database/models/entry-related.ts b/src/renderer/src/database/models/entry-related.ts deleted file mode 100644 index 3c4531ff8e..0000000000 --- a/src/renderer/src/database/models/entry-related.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { BaseModel } from "../model" -import { DB_BaseSchema } from "../schemas" - -class ModelStatic extends BaseModel<"entryRelated"> { - constructor() { - super("entryRelated", DB_BaseSchema) - } -} - -export const entryRelatedModel = new ModelStatic() diff --git a/src/renderer/src/database/models/entry.ts b/src/renderer/src/database/models/entry.ts deleted file mode 100644 index 1d8e58aa91..0000000000 --- a/src/renderer/src/database/models/entry.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { BaseModel } from "../model" -import { DB_BaseSchema } from "../schemas" - -class EntryModelStatic extends BaseModel<"entries"> { - constructor() { - super("entries", DB_BaseSchema) - } -} - -export const entryModel = new EntryModelStatic() diff --git a/src/renderer/src/database/models/feed-entry.ts b/src/renderer/src/database/models/feed-entry.ts deleted file mode 100644 index 825b606bc1..0000000000 --- a/src/renderer/src/database/models/feed-entry.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { BaseModel } from "../model" -import { DB_BaseSchema } from "../schemas" - -class ModelStatic extends BaseModel<"feedEntries"> { - constructor() { - super("feedEntries", DB_BaseSchema) - } -} - -export const feedEntriesModel = new ModelStatic() diff --git a/src/renderer/src/database/models/feed.ts b/src/renderer/src/database/models/feed.ts deleted file mode 100644 index 9352cc457d..0000000000 --- a/src/renderer/src/database/models/feed.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { BaseModel } from "../model" -import { DB_BaseSchema } from "../schemas" - -class ModelStatic extends BaseModel<"feeds"> { - constructor() { - super("feeds", DB_BaseSchema) - } -} - -export const feedModel = new ModelStatic() diff --git a/src/renderer/src/database/models/index.ts b/src/renderer/src/database/models/index.ts deleted file mode 100644 index 4e626bebfd..0000000000 --- a/src/renderer/src/database/models/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from "./entry" -export * from "./entry-related" -export * from "./feed" -export * from "./feed-entry" -export * from "./subscription" diff --git a/src/renderer/src/database/models/subscription.ts b/src/renderer/src/database/models/subscription.ts deleted file mode 100644 index c0f83dc35c..0000000000 --- a/src/renderer/src/database/models/subscription.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { BaseModel } from "../model" -import { DB_BaseSchema } from "../schemas" - -class ModelStatic extends BaseModel<"subscriptions"> { - constructor() { - super("subscriptions", DB_BaseSchema) - } -} - -export const subscriptionModel = new ModelStatic() diff --git a/src/renderer/src/database/models/unread.ts b/src/renderer/src/database/models/unread.ts deleted file mode 100644 index 9124451f60..0000000000 --- a/src/renderer/src/database/models/unread.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { BaseModel } from "../model" -import { DB_BaseSchema } from "../schemas" - -class ModelStatic extends BaseModel<"feedUnreads"> { - constructor() { - super("feedUnreads", DB_BaseSchema) - } -} - -export const feedUnreadModel = new ModelStatic() diff --git a/src/renderer/src/database/models/user.ts b/src/renderer/src/database/models/user.ts deleted file mode 100644 index 00056a9243..0000000000 --- a/src/renderer/src/database/models/user.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type { users } from "@renderer/hono" - -export type UserModel = Omit diff --git a/src/renderer/src/database/schemas/cleaner.ts b/src/renderer/src/database/schemas/cleaner.ts new file mode 100644 index 0000000000..a3e6ecad4c --- /dev/null +++ b/src/renderer/src/database/schemas/cleaner.ts @@ -0,0 +1,5 @@ +export type DB_Cleaner = { + refId: string + visitedAt: number + type: "entry" | "feed" +} diff --git a/src/renderer/src/database/schemas/entry.ts b/src/renderer/src/database/schemas/entry.ts new file mode 100644 index 0000000000..5092906f67 --- /dev/null +++ b/src/renderer/src/database/schemas/entry.ts @@ -0,0 +1,8 @@ +import type { EntryModel } from "@renderer/models" + +export type DB_Entry = EntryModel & { feedId: string } + +export type DB_EntryRelated = { + id: string + data: any +} diff --git a/src/renderer/src/database/schemas/feed.ts b/src/renderer/src/database/schemas/feed.ts index f729d3dbbd..489611e39f 100644 --- a/src/renderer/src/database/schemas/feed.ts +++ b/src/renderer/src/database/schemas/feed.ts @@ -1,7 +1,8 @@ -import { z } from "zod" +import type { FeedModel } from "@renderer/models" -export const DB_FeedIdSchema = z.object({ - feedId: z.string(), -}) +export type DB_FeedUnread = { + id: string + count: number +} -export type DB_FeedId = z.infer +export type DB_Feed = FeedModel & { id: string } diff --git a/src/renderer/src/database/schemas/subscription.ts b/src/renderer/src/database/schemas/subscription.ts new file mode 100644 index 0000000000..ae7f99b365 --- /dev/null +++ b/src/renderer/src/database/schemas/subscription.ts @@ -0,0 +1,5 @@ +import type { SubscriptionFlatModel } from "@renderer/store/subscription" + +export type DB_Subscription = SubscriptionFlatModel & { + id: string +} diff --git a/src/renderer/src/database/types.ts b/src/renderer/src/database/types.ts deleted file mode 100644 index cc4279c9ea..0000000000 --- a/src/renderer/src/database/types.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type DBModel = T & { - createdAt: number - id: string - updatedAt: number -} diff --git a/src/renderer/src/initialize/hydrate.ts b/src/renderer/src/initialize/hydrate.ts index 4a67055e5b..0362476912 100644 --- a/src/renderer/src/initialize/hydrate.ts +++ b/src/renderer/src/initialize/hydrate.ts @@ -71,7 +71,7 @@ async function hydrateEntry() { const storeValue = [] as FlatEntryModel[] for (const entry of entries) { - const entryRelatedFeedId = feedEntries[entry.id] + const entryRelatedFeedId = entry.feedId || feedEntries[entry.id] if (!entryRelatedFeedId) { logHydrateError(`Entry ${entry.id} has no related feed id`) continue diff --git a/src/renderer/src/initialize/index.ts b/src/renderer/src/initialize/index.ts index c5395a4ff9..5f0e1c9a88 100644 --- a/src/renderer/src/initialize/index.ts +++ b/src/renderer/src/initialize/index.ts @@ -2,6 +2,7 @@ import { env } from "@env" import { authConfigManager } from "@hono/auth-js/react" import { repository } from "@pkg" import { browserDB } from "@renderer/database" +import { CleanerService } from "@renderer/services/cleaner" import { registerGlobalContext } from "@shared/bridge" import { enableMapSet } from "immer" import { toast } from "sonner" @@ -62,6 +63,7 @@ export const initializeApp = async () => { // Initialize the database if (enabledDataPersist) { dataHydratedTime = await hydrateDatabaseToStore() + CleanerService.cleanOutdatedData() } // Initialize the auth config diff --git a/src/renderer/src/models/types.ts b/src/renderer/src/models/types.ts index 74e25d29b0..58a2025b50 100644 --- a/src/renderer/src/models/types.ts +++ b/src/renderer/src/models/types.ts @@ -1,5 +1,8 @@ +import type { users } from "@renderer/hono" import type { apiClient } from "@renderer/lib/api-fetch" +export type UserModel = Omit + export type ExtractBizResponse any> = Exclude< Awaited>, undefined diff --git a/src/renderer/src/providers/user-provider.tsx b/src/renderer/src/providers/user-provider.tsx index 218cf7207b..0890261bc4 100644 --- a/src/renderer/src/providers/user-provider.tsx +++ b/src/renderer/src/providers/user-provider.tsx @@ -1,5 +1,6 @@ import { useSetMe } from "@renderer/atoms/user" import { useSession } from "@renderer/queries/auth" +import { CleanerService } from "@renderer/services/cleaner" import { useEffect } from "react" export const UserProvider = () => { @@ -13,6 +14,8 @@ export const UserProvider = () => { name: session.user.name, handle: session.user.handle, }) + + CleanerService.cleanRemainingData(session.user.id) }, [session?.user, setUser]) return null diff --git a/src/renderer/src/services/__snapshots__/cleaner.spec.ts.snap b/src/renderer/src/services/__snapshots__/cleaner.spec.ts.snap new file mode 100644 index 0000000000..d1f1af8a04 --- /dev/null +++ b/src/renderer/src/services/__snapshots__/cleaner.spec.ts.snap @@ -0,0 +1,85 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`test db cleaner > data should be set up correctly 1`] = ` +[ + { + "id": "feed-id-1", + }, + { + "id": "feed-id-2", + }, + { + "id": "feed-id-3", + }, + { + "id": "feed-id-4", + }, + { + "id": "feed-id-5", + }, +] +`; + +exports[`test db cleaner > data should be set up correctly 2`] = ` +[ + { + "feedId": "feed-id-1", + "id": "entry-id-1", + }, + { + "feedId": "feed-id-2", + "id": "entry-id-2", + }, + { + "feedId": "feed-id-3", + "id": "entry-id-3", + }, + { + "feedId": "feed-id-4", + "id": "entry-id-4", + }, + { + "feedId": "feed-id-5", + "id": "entry-id-5", + }, + { + "feedId": "feed-id-1", + "id": "entry-id-6", + }, +] +`; + +exports[`test db cleaner > data should be set up correctly 3`] = ` +[ + { + "feedId": "feed-id-1", + "id": "test-user-id-1/feed-id-1", + "userId": "test-user-id-1", + }, + { + "feedId": "feed-id-3", + "id": "test-user-id-1/feed-id-3", + "userId": "test-user-id-1", + }, + { + "feedId": "feed-id-4", + "id": "test-user-id-2/feed-id-4", + "userId": "test-user-id-2", + }, + { + "feedId": "feed-id-5", + "id": "test-user-id-3/feed-id-5", + "userId": "test-user-id-3", + }, + { + "feedId": "feed-id-1", + "id": "test-user-id/feed-id-1", + "userId": "test-user-id", + }, + { + "feedId": "feed-id-2", + "id": "test-user-id/feed-id-2", + "userId": "test-user-id", + }, +] +`; diff --git a/src/renderer/src/services/base.ts b/src/renderer/src/services/base.ts index bd7aa14fd3..7a51d12338 100644 --- a/src/renderer/src/services/base.ts +++ b/src/renderer/src/services/base.ts @@ -4,14 +4,6 @@ import type { UpdateSpec } from "dexie" export abstract class BaseService { constructor(public readonly table: Dexie.Table) {} - async create(data: T) { - return this.table.add(data) - } - - async insertMany(data: T[]) { - return this.table.bulkAdd(data) - } - async upsert(data: T): Promise { return this.table.put(data) } diff --git a/src/renderer/src/services/cleaner.spec.ts b/src/renderer/src/services/cleaner.spec.ts new file mode 100644 index 0000000000..2a307b4fc0 --- /dev/null +++ b/src/renderer/src/services/cleaner.spec.ts @@ -0,0 +1,157 @@ +// @ts-nocheck +import { browserDB } from "@renderer/database" +import type { FeedModel } from "@renderer/hono" +import type { EntryModel } from "@renderer/models" +import type { SubscriptionFlatModel } from "@renderer/store/subscription" +import { beforeAll, describe, expect, test } from "vitest" + +import { CleanerService } from "./cleaner" +import { EntryService } from "./entry" +import { FeedService } from "./feed" +import { SubscriptionService } from "./subscription" + +const currentUserID = "test-user-id" +const otherUserIDs = ["test-user-id-1", "test-user-id-2", "test-user-id-3"] + +const subscriptions: SubscriptionFlatModel[] = [ + { + feedId: "feed-id-1", + userId: currentUserID, + }, + { + feedId: "feed-id-2", + userId: currentUserID, + }, + { + feedId: "feed-id-3", + userId: otherUserIDs[0], + }, + { + feedId: "feed-id-4", + userId: otherUserIDs[1], + }, + { + feedId: "feed-id-5", + userId: otherUserIDs[2], + }, + // ==== + { + feedId: "feed-id-1", + userId: otherUserIDs[0], + }, +] + +const feeds: FeedModel[] = [ + { + id: "feed-id-1", + }, + { + id: "feed-id-2", + }, + { + id: "feed-id-3", + }, + { + id: "feed-id-4", + }, + { + id: "feed-id-5", + }, +] + +const entries: EntryModel[] = [ + { + id: "entry-id-1", + }, + { + id: "entry-id-2", + }, + { + id: "entry-id-3", + }, + { + id: "entry-id-4", + }, + { + id: "entry-id-5", + }, + { + id: "entry-id-6", + }, +] + +const entryFeedIdMap = { + "entry-id-1": "feed-id-1", + "entry-id-2": "feed-id-2", + "entry-id-3": "feed-id-3", + "entry-id-4": "feed-id-4", + "entry-id-5": "feed-id-5", + "entry-id-6": "feed-id-1", +} +describe("test db cleaner", () => { + beforeAll(async () => { + await browserDB.delete() + }) + beforeAll(async () => { + await browserDB.open() + await SubscriptionService.upsertMany(subscriptions) + await FeedService.upsertMany(feeds) + await EntryService.upsertMany(entries, entryFeedIdMap) + }) + + test("data should be set up correctly", async () => { + const feeds = await FeedService.findAll() + const entries = await EntryService.findAll() + const subscriptions = await SubscriptionService.findAll() + expect(feeds).toMatchSnapshot() + expect(entries).toMatchSnapshot() + expect(subscriptions).toMatchSnapshot() + }) + test("should clean remaining data", async () => { + await CleanerService.cleanRemainingData(currentUserID) + + const feeds = await FeedService.findAll() + expect(feeds).toMatchInlineSnapshot(` + [ + { + "id": "feed-id-1", + }, + { + "id": "feed-id-2", + }, + ] + `) + const subscripions = await SubscriptionService.findAll() + expect(subscripions).toMatchInlineSnapshot(` + [ + { + "feedId": "feed-id-1", + "id": "test-user-id/feed-id-1", + "userId": "test-user-id", + }, + { + "feedId": "feed-id-2", + "id": "test-user-id/feed-id-2", + "userId": "test-user-id", + }, + ] + `) + const entries = await EntryService.findAll() + expect(entries).toMatchInlineSnapshot(` + [ + { + "feedId": "feed-id-1", + "id": "entry-id-1", + }, + { + "feedId": "feed-id-2", + "id": "entry-id-2", + }, + { + "feedId": "feed-id-1", + "id": "entry-id-6", + }, + ] + `) + }) +}) diff --git a/src/renderer/src/services/cleaner.ts b/src/renderer/src/services/cleaner.ts new file mode 100644 index 0000000000..5d8c5e52ca --- /dev/null +++ b/src/renderer/src/services/cleaner.ts @@ -0,0 +1,90 @@ +import { browserDB } from "@renderer/database" +import { appLog } from "@renderer/lib/log" + +import { EntryService } from "./entry" +import { FeedService } from "./feed" +import { FeedUnreadService } from "./feed-unread" +import { SubscriptionService } from "./subscription" + +const cleanerModel = browserDB.cleaner +class CleanerServiceStatic { + // Clean other user subscriptions, should call this after login + async cleanRemainingData(currentUserId: string) { + const dbUserIds = await SubscriptionService.getUserIds() + + const otherUserIds = dbUserIds.filter((id) => id !== currentUserId) + const remainingSubscriptions = + await SubscriptionService.getUserSubscriptions(otherUserIds) + const currentSubscription = await SubscriptionService.getUserSubscriptions([ + currentUserId, + ]) + const currentSubscriptionFeedsSet = new Set( + currentSubscription.map((s) => s.feedId), + ) + // Finds a subscripion that does not exist for the current user, but a feedId that exists in the db + + const toRemoveSubscription = remainingSubscriptions.filter( + (s) => !currentSubscriptionFeedsSet.has(s.feedId), + ) + + const toRemoveFeedIds = toRemoveSubscription.map((s) => s.feedId) + + await Promise.allSettled([ + otherUserIds.map((id) => SubscriptionService.removeSubscription(id)), + FeedService.bulkDelete(toRemoveFeedIds), + FeedUnreadService.bulkDelete(toRemoveFeedIds), + EntryService.deleteEntriesByFeedIds(toRemoveFeedIds), + ]) + } + + /** + * Mark the which data recently used + */ + renew(list: { type: "feed" | "entry", id: string }[]) { + const now = Date.now() + return cleanerModel.bulkPut( + list.map((item) => ({ + refId: item.id, + visitedAt: now, + type: item.type, + })), + ) + } + + /** + * Remove the data that not used for a long time + */ + async cleanOutdatedData() { + const now = Date.now() + const expiredTime = now - 1000 * 60 * 60 * 24 * 30 // 30 days + const data = await cleanerModel + .where("visitedAt") + .below(expiredTime) + .toArray() + + if (data.length === 0) { return } + const deleteEntries = [] as string[] + const deleteFeeds = [] as string[] + for (const item of data) { + switch (item.type) { + case "feed": { + deleteFeeds.push(item.refId) + break + } + case "entry": { + deleteEntries.push(item.refId) + break + } + } + } + + appLog("Clean outdated data...", "feeds:", deleteFeeds.length, "entries:", deleteEntries.length) + await Promise.allSettled([ + FeedService.bulkDelete(deleteFeeds), + EntryService.deleteEntries(deleteEntries), + EntryService.deleteEntriesByFeedIds(deleteFeeds), + cleanerModel.bulkDelete(data.map((d) => d.refId)), + ]) + } +} +export const CleanerService = new CleanerServiceStatic() diff --git a/src/renderer/src/services/entry-related.ts b/src/renderer/src/services/entry-related.ts index f7d85c64dd..56be57cd19 100644 --- a/src/renderer/src/services/entry-related.ts +++ b/src/renderer/src/services/entry-related.ts @@ -1,7 +1,8 @@ -import { entryRelatedModel } from "@renderer/database" +import { browserDB } from "@renderer/database" export enum EntryRelatedKey { READ = "READ", + /** @deprecated */ FEED_ID = "FEED_ID", COLLECTION = "COLLECTION", } @@ -17,13 +18,14 @@ const taskQueue = new Map>( type IdToIdRecord = Record type IdToBooleanRecord = Record type IdToAnyObjectRecord = Record> +const entryRelatedModel = browserDB.entryRelated class ServiceStatic { async findAll(type: EntryRelatedKey.FEED_ID): Promise async findAll(type: EntryRelatedKey.READ): Promise async findAll(type: EntryRelatedKey.COLLECTION): Promise async findAll(type: EntryRelatedKey): Promise> { - const data = await entryRelatedModel.table.get(type) + const data = await entryRelatedModel.get(type) return data ? data.data : {} } @@ -50,7 +52,7 @@ class ServiceStatic { const task = getPreviousTask.finally(async () => { const oldData = await this.findAll(type) - entryRelatedModel.table.put({ + entryRelatedModel.put({ data: { ...oldData, ...data }, id: type, }) @@ -64,14 +66,14 @@ class ServiceStatic { const oldData = await this.findAll(type as any) delete oldData[key] - return entryRelatedModel.table.put({ + return entryRelatedModel.put({ data: oldData, id: type, }) } async clear() { - return entryRelatedModel.table.clear() + return entryRelatedModel.clear() } } diff --git a/src/renderer/src/services/entry.ts b/src/renderer/src/services/entry.ts index 097769bf3e..db7e7b8c12 100644 --- a/src/renderer/src/services/entry.ts +++ b/src/renderer/src/services/entry.ts @@ -1,7 +1,8 @@ -import { entryModel } from "@renderer/database/models" +import { browserDB } from "@renderer/database" import type { EntryModel } from "@renderer/models/types" import { BaseService } from "./base" +import { CleanerService } from "./cleaner" import { EntryRelatedKey, EntryRelatedService } from "./entry-related" type EntryCollection = { @@ -9,13 +10,59 @@ type EntryCollection = { } class EntryServiceStatic extends BaseService { constructor() { - super(entryModel.table) + super(browserDB.entries) + } + + // @ts-expect-error + override async upsertMany( + data: EntryModel[], + entryFeedMap: Record, + ) { + const renewList = [] as { type: "entry", id: string }[] + const nextData = [] as (EntryModel & { feedId: string })[] + + for (const item of data) { + const feedId = entryFeedMap[item.id] + if (!feedId) { + console.error("EntryService.upsertMany: feedId not found", item) + continue + } + renewList.push({ type: "entry", id: item.id }) + nextData.push({ + ...item, + feedId, + }) + } + + CleanerService.renew(renewList) + + return super.upsertMany(nextData) + } + + // @ts-ignore + override async upsert(feedId: string, data: EntryModel): Promise { + CleanerService.renew([ + { + type: "entry", + id: data.id, + }, + ]) + return super.upsert({ + ...data, + // @ts-expect-error + feedId, + }) + } + + override async findAll() { + return super.findAll() as Promise<(EntryModel & { feedId: string })[]> } bulkStoreReadStatus(record: Record) { return EntryRelatedService.upsert(EntryRelatedKey.READ, record) } + /** @deprecated */ bulkStoreFeedId(record: Record) { return EntryRelatedService.upsert(EntryRelatedKey.FEED_ID, record) } @@ -29,7 +76,11 @@ class EntryServiceStatic extends BaseService { } async deleteEntries(entryIds: string[]) { - await entryModel.table.bulkDelete(entryIds) + await this.table.bulkDelete(entryIds) + } + + async deleteEntriesByFeedIds(feedIds: string[]) { + await this.table.where("feedId").anyOf(feedIds).delete() } } diff --git a/src/renderer/src/services/feed-entry.ts b/src/renderer/src/services/feed-entry.ts deleted file mode 100644 index c1880bd5eb..0000000000 --- a/src/renderer/src/services/feed-entry.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { feedEntriesModel } from "@renderer/database" - -class ServiceStatic { - constructor() {} - - updateFeed(feedId: string, entryIds: string[]) { - return feedEntriesModel.table.put({ - feedId, - entryIds, - }) - } - - getAll() { - return feedEntriesModel.table.toArray() - } -} -/** - * Not used yet - */ -export const FeedEntryService = new ServiceStatic() diff --git a/src/renderer/src/services/feed-unread.ts b/src/renderer/src/services/feed-unread.ts index 7b6454df24..1b30aa3dc2 100644 --- a/src/renderer/src/services/feed-unread.ts +++ b/src/renderer/src/services/feed-unread.ts @@ -1,14 +1,15 @@ -import { feedUnreadModel } from "@renderer/database/models/unread" +import { browserDB } from "@renderer/database" +const feedUnreadModel = browserDB.feedUnreads class ServiceStatic { updateFeedUnread(list: [string, number][]) { - return feedUnreadModel.table.bulkPut( + return feedUnreadModel.bulkPut( list.map(([feedId, count]) => ({ id: feedId, count })), ) } getAll() { - return feedUnreadModel.table.toArray() as Promise< + return feedUnreadModel.toArray() as Promise< { id: string count: number @@ -17,7 +18,11 @@ class ServiceStatic { } clear() { - return feedUnreadModel.table.clear() + return feedUnreadModel.clear() + } + + async bulkDelete(ids: string[]) { + return feedUnreadModel.bulkDelete(ids) } } diff --git a/src/renderer/src/services/feed.ts b/src/renderer/src/services/feed.ts index fcce92f01a..358f809d69 100644 --- a/src/renderer/src/services/feed.ts +++ b/src/renderer/src/services/feed.ts @@ -1,23 +1,31 @@ -import { feedModel } from "@renderer/database/models" +import { browserDB } from "@renderer/database" import type { FeedModel } from "@renderer/models/types" import { BaseService } from "./base" +import { CleanerService } from "./cleaner" type FeedModelWithId = FeedModel & { id: string } class ServiceStatic extends BaseService { constructor() { - super(feedModel.table) + super(browserDB.feeds) } override async upsertMany(data: FeedModel[]) { const filterData = data.filter((d) => d.id) + CleanerService.renew(filterData.map((d) => ({ type: "feed", id: d.id! }))) + return this.table.bulkPut(filterData as FeedModelWithId[]) } override async upsert(data: FeedModel): Promise { if (!data.id) return null + CleanerService.renew([{ type: "feed", id: data.id }]) return this.table.put(data as FeedModelWithId) } + + async bulkDelete(ids: string[]) { + return this.table.bulkDelete(ids) + } } export const FeedService = new ServiceStatic() diff --git a/src/renderer/src/services/index.ts b/src/renderer/src/services/index.ts index 7939d25e17..835beafbb4 100644 --- a/src/renderer/src/services/index.ts +++ b/src/renderer/src/services/index.ts @@ -1,6 +1,5 @@ export * from "./entry" export * from "./entry-related" export * from "./feed" -export * from "./feed-entry" export * from "./feed-unread" export * from "./subscription" diff --git a/src/renderer/src/services/subscription.ts b/src/renderer/src/services/subscription.ts index 0a9c690dff..027076e7ae 100644 --- a/src/renderer/src/services/subscription.ts +++ b/src/renderer/src/services/subscription.ts @@ -1,5 +1,6 @@ -import { subscriptionModel } from "@renderer/database" +import { browserDB } from "@renderer/database" import type { SubscriptionFlatModel } from "@renderer/store/subscription" +import { uniq } from "lodash-es" import { BaseService } from "./base" @@ -7,7 +8,22 @@ type SubscriptionModelWithId = SubscriptionFlatModel & { id: string } class SubscriptionServiceStatic extends BaseService { constructor() { - super(subscriptionModel.table) + super(browserDB.subscriptions) + } + + public getUserSubscriptions(userIds: string[]) { + return this.table.where("userId").anyOf(userIds).toArray() + } + + public async getUserIds() { + return uniq( + (await this.table + .toCollection() + .uniqueKeys() + .then((keys) => + keys.map((k) => k.toString().split("/")[0]), + )) as string[], + ) } override async upsertMany(data: SubscriptionFlatModel[]) { @@ -33,6 +49,18 @@ class SubscriptionServiceStatic extends BaseService { async changeView(feedId: string, view: number) { return this.table.where("feedId").equals(feedId).modify({ view }) } + + async removeSubscription(userId: string, feedId: string): Promise + // @ts-expect-error + async removeSubscription(userId: string): Promise + async removeSubscription(userId: string, feedId: string) { + if (feedId && userId) { + return this.table.delete(this.uniqueId(userId, feedId)) + } + if (!feedId && userId) { + return this.table.where("userId").equals(userId).delete() + } + } } export const SubscriptionService = new SubscriptionServiceStatic() diff --git a/src/renderer/src/store/entry/store.ts b/src/renderer/src/store/entry/store.ts index 29bd71e732..a1498bb505 100644 --- a/src/renderer/src/store/entry/store.ts +++ b/src/renderer/src/store/entry/store.ts @@ -1,15 +1,11 @@ import { runTransactionInScope } from "@renderer/database" -import type { UserModel } from "@renderer/database/models/user" import type { EntryReadHistoriesModel } from "@renderer/hono" import { apiClient } from "@renderer/lib/api-fetch" import { getEntriesParams, omitObjectUndefinedValue, } from "@renderer/lib/utils" -import type { - CombinedEntryModel, - EntryModel, - FeedModel, +import type { CombinedEntryModel, EntryModel, FeedModel, UserModel, } from "@renderer/models" import { EntryService } from "@renderer/services" import { produce } from "immer" @@ -225,9 +221,8 @@ class EntryActions { // Update database runTransactionInScope(() => Promise.all([ - EntryService.upsertMany(entries), + EntryService.upsertMany(entries, entryFeedMap), EntryService.bulkStoreReadStatus(entry2Read), - EntryService.bulkStoreFeedId(entryFeedMap), EntryService.bulkStoreCollection(entryCollection), ]), ) diff --git a/src/renderer/src/store/user/hooks.ts b/src/renderer/src/store/user/hooks.ts index e45523f756..c97891a5c4 100644 --- a/src/renderer/src/store/user/hooks.ts +++ b/src/renderer/src/store/user/hooks.ts @@ -1,4 +1,4 @@ -import type { UserModel } from "@renderer/database/models/user" +import type { UserModel } from "@renderer/models" import { useUserStore } from "./store" diff --git a/src/renderer/src/store/user/store.ts b/src/renderer/src/store/user/store.ts index fd231ebac1..abde637ca7 100644 --- a/src/renderer/src/store/user/store.ts +++ b/src/renderer/src/store/user/store.ts @@ -1,4 +1,4 @@ -import type { UserModel } from "@renderer/database/models/user" +import type { UserModel } from "@renderer/models" import { produce } from "immer" import { createZustandStore } from "../utils/helper" diff --git a/src/renderer/src/store/utils/clear.ts b/src/renderer/src/store/utils/clear.ts index 75c190f352..9caf5a7b98 100644 --- a/src/renderer/src/store/utils/clear.ts +++ b/src/renderer/src/store/utils/clear.ts @@ -8,14 +8,11 @@ import { feedUnreadActions } from "../unread" export const clearLocalPersistStoreData = () => { // All clear and reset method will aggregate here - [ - entryActions, - subscriptionActions, - feedUnreadActions, - feedActions, - ].forEach((actions) => { - actions.clear() - }) + [entryActions, subscriptionActions, feedUnreadActions, feedActions].forEach( + (actions) => { + actions.clear() + }, + ) clearUISettings() diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000000..f9c9372511 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,36 @@ +import { readFileSync } from "node:fs" +import { resolve } from "node:path" +import { fileURLToPath } from "node:url" + +import tsconfigPath from "vite-tsconfig-paths" +import { defineConfig } from "vitest/config" + +const pkg = JSON.parse(readFileSync("package.json", "utf8")) +const __dirname = fileURLToPath(new URL(".", import.meta.url)) +export default defineConfig({ + root: "./", + test: { + include: ["**/*.test.ts", "**/*.spec.ts"], + + globals: true, + setupFiles: [resolve(__dirname, "./setup-file.ts")], + environment: "node", + includeSource: [resolve(__dirname, ".")], + }, + + define: { + APP_VERSION: JSON.stringify(pkg.version), + APP_NAME: JSON.stringify(pkg.name), + APP_DEV_CWD: JSON.stringify(process.cwd()), + + GIT_COMMIT_SHA: "'SHA'", + DEBUG: process.env.DEBUG === "true", + ELECTRON: "false", + }, + + plugins: [ + tsconfigPath({ + projects: [resolve(__dirname, "./tsconfig.json")], + }), + ], +})