server: remove unused history snapshots and make it in-mem only

jesec · Nov 1, 2021 · af8de75 · af8de75
1 parent 0cbd390
commit af8de75
Show file tree

Hide file tree

Showing 10 changed files with 61 additions and 205 deletions.
diff --git a/client/src/javascript/actions/FloodActions.ts b/client/src/javascript/actions/FloodActions.ts
@@ -10,18 +10,12 @@ import TransferDataStore from '@client/stores/TransferDataStore';
 import UIStore from '@client/stores/UIStore';
 
 import type {DirectoryListResponse} from '@shared/types/api';
-import type {HistorySnapshot} from '@shared/constants/historySnapshotTypes';
 import type {NotificationFetchOptions, NotificationState} from '@shared/types/Notification';
 import type {ServerEvents} from '@shared/types/ServerEvents';
 
-interface ActivityStreamOptions {
-  historySnapshot: HistorySnapshot;
-}
-
 const {baseURI} = ConfigStore;
 
 let activityStreamEventSource: EventSource | null = null;
-let lastActivityStreamOptions: ActivityStreamOptions;
 let visibilityChangeTimeout: NodeJS.Timeout;
 
 // TODO: Use standard Event interfaces
@@ -120,26 +114,14 @@ const FloodActions = {
 
   restartActivityStream() {
     this.closeActivityStream();
-    this.startActivityStream(lastActivityStreamOptions);
+    this.startActivityStream();
   },
 
-  startActivityStream(options: ActivityStreamOptions = {historySnapshot: 'FIVE_MINUTE'}) {
-    const {historySnapshot} = options;
-    const didHistorySnapshotChange =
-      lastActivityStreamOptions && lastActivityStreamOptions.historySnapshot !== historySnapshot;
-
-    lastActivityStreamOptions = options;
-
-    // When the user requests a new history snapshot during an open session,
-    // we need to close and re-open the event stream.
-    if (didHistorySnapshotChange && activityStreamEventSource != null) {
-      this.closeActivityStream();
-    }
-
+  startActivityStream() {
     // If the user requested a new history snapshot, or the event source has not
     // alraedy been created, we open the event stream.
-    if (didHistorySnapshotChange || activityStreamEventSource == null) {
-      activityStreamEventSource = new EventSource(`${baseURI}api/activity-stream?historySnapshot=${historySnapshot}`);
+    if (activityStreamEventSource == null) {
+      activityStreamEventSource = new EventSource(`${baseURI}api/activity-stream`);
 
       Object.entries(ServerEventHandlers).forEach(([event, handler]) => {
         if (activityStreamEventSource != null) {
@@ -162,7 +144,7 @@ const handleWindowVisibilityChange = () => {
     global.clearTimeout(visibilityChangeTimeout);
 
     if (activityStreamEventSource == null) {
-      FloodActions.startActivityStream(lastActivityStreamOptions);
+      FloodActions.startActivityStream();
     }
   }
 };

diff --git a/server/bin/migrations/UserInDatabase2.ts → server/bin/migrations/00-UserInDatabase2.ts b/server/bin/migrations/UserInDatabase2.ts → server/bin/migrations/00-UserInDatabase2.ts
diff --git a/server/bin/migrations/UserInDatabase3.ts → server/bin/migrations/01-UserInDatabase3.ts b/server/bin/migrations/UserInDatabase3.ts → server/bin/migrations/01-UserInDatabase3.ts
diff --git a/server/bin/migrations/02-HistoryEra.ts b/server/bin/migrations/02-HistoryEra.ts
@@ -0,0 +1,17 @@
+import fs from 'fs-extra';
+import path from 'path';
+
+import config from '../../../config';
+import Users from '../../models/Users';
+
+const migration = () => {
+  return Users.listUsers().then((users) => {
+    return Promise.all(
+      users.map((user) => fs.rm(path.join(config.dbPath, user._id, 'history'), {recursive: true})),
+    ).catch(() => {
+      // do nothing.
+    });
+  });
+};
+
+export default migration;
diff --git a/server/bin/migrations/run.ts b/server/bin/migrations/run.ts
@@ -1,8 +1,13 @@
-import UserInDatabase2 from './UserInDatabase2';
-import UserInDatabase3 from './UserInDatabase3';
+import UserInDatabase2 from './00-UserInDatabase2';
+import UserInDatabase3 from './01-UserInDatabase3';
+import HistoryEra from './02-HistoryEra';
 
-const migrations = [UserInDatabase2, UserInDatabase3];
+const migrations = [UserInDatabase2, UserInDatabase3, HistoryEra];
 
-const migrate = () => Promise.all(migrations.map((migration) => migration()));
+const migrate = async () => {
+  for await (const migrate of migrations) {
+    await migrate();
+  }
+};
 
 export default migrate;
diff --git a/server/middleware/clientActivityStream.ts b/server/middleware/clientActivityStream.ts
@@ -2,20 +2,15 @@ import type {Operation} from 'fast-json-patch';
 import type {Request, Response} from 'express';
 import type TypedEmitter from 'typed-emitter';
 
-import type {HistorySnapshot} from '@shared/constants/historySnapshotTypes';
-
 import DiskUsage from '../models/DiskUsage';
 import {getAllServices} from '../services';
 import ServerEvent from '../models/ServerEvent';
 
 import type {DiskUsageSummary} from '../models/DiskUsage';
 import type {TransferHistory} from '../../shared/types/TransferData';
 
-export default async (req: Request<unknown, unknown, unknown, {historySnapshot: HistorySnapshot}>, res: Response) => {
-  const {
-    query: {historySnapshot = 'FIVE_MINUTE'},
-    user,
-  } = req;
+export default async (req: Request, res: Response) => {
+  const {user} = req;
 
   if (user == null) {
     return;
@@ -60,7 +55,7 @@ export default async (req: Request<unknown, unknown, unknown, {historySnapshot:
   }
 
   // Transfer history
-  await serviceInstances.historyService.getHistory({snapshot: historySnapshot}).then(
+  await serviceInstances.historyService.getHistory().then(
     (snapshot) => {
       const {timestamps: lastTimestamps} = snapshot || {timestamps: []};
       const lastTimestamp = lastTimestamps[lastTimestamps.length - 1];

diff --git a/server/models/HistoryEra.ts b/server/models/HistoryEra.ts
@@ -1,108 +1,45 @@
-import type {UserInDatabase} from '@shared/schema/Auth';
 import type {TransferData, TransferSnapshot} from '@shared/types/TransferData';
 
 import Datastore from 'nedb-promises';
-import path from 'path';
+import {setInterval} from 'timers';
 
 import config from '../../config';
 
 interface HistoryEraOpts {
   interval: number;
   maxTime: number;
   name: string;
-  nextEraUpdateInterval?: number;
-  nextEra?: HistoryEra;
 }
 
-const MAX_NEXT_ERA_UPDATE_INTERVAL = 1000 * 60 * 60 * 12; // 12 hours
 const CUMULATIVE_DATA_BUFFER_DIFF = 500; // 500 milliseconds
 
 class HistoryEra {
-  data = [];
-  ready: Promise<void>;
-  lastUpdate = 0;
-  startedAt = Date.now();
-  opts: HistoryEraOpts;
-  db: Datastore;
-  autoCleanupInterval?: NodeJS.Timeout;
-  nextEraUpdateInterval?: NodeJS.Timeout;
-
-  constructor(user: UserInDatabase, opts: HistoryEraOpts) {
-    this.opts = opts;
-    this.db = Datastore.create({
-      autoload: true,
-      filename: path.join(config.dbPath, user._id, 'history', `${opts.name}.db`),
-    });
-    this.ready = this.prepareDatabase();
-  }
-
-  private async prepareDatabase(): Promise<void> {
-    let lastUpdate = 0;
-
-    await this.db.find<TransferSnapshot>({}).then(
-      (snapshots) => {
-        snapshots.forEach((snapshot) => {
-          if (snapshot.timestamp > lastUpdate) {
-            lastUpdate = snapshot.timestamp;
-          }
-        });
+  private lastUpdate = 0;
+  private opts: HistoryEraOpts;
+  private db: Datastore;
 
-        this.lastUpdate = lastUpdate;
-      },
-      () => undefined,
-    );
-
-    await this.removeOutdatedData();
+  constructor(opts: HistoryEraOpts) {
+    this.opts = opts;
+    this.db = Datastore.create();
 
     let cleanupInterval = this.opts.maxTime;
 
     if (cleanupInterval === 0 || cleanupInterval > config.dbCleanInterval) {
       cleanupInterval = config.dbCleanInterval;
     }
 
-    this.autoCleanupInterval = setInterval(this.removeOutdatedData, cleanupInterval);
+    setInterval(this.removeOutdatedData, cleanupInterval);
   }
 
-  private removeOutdatedData = async (): Promise<void> => {
-    if (this.opts.maxTime > 0) {
-      const minTimestamp = Date.now() - this.opts.maxTime;
-      return this.db.remove({timestamp: {$lt: minTimestamp}}, {multi: true}).then(
-        () => undefined,
-        () => undefined,
-      );
-    }
-  };
-
-  private updateNextEra = async (): Promise<void> => {
-    if (this.opts.nextEraUpdateInterval == null) {
-      return;
-    }
-
-    const minTimestamp = Date.now() - this.opts.nextEraUpdateInterval;
-
-    return this.db.find<TransferSnapshot>({timestamp: {$gte: minTimestamp}}).then((snapshots) => {
-      if (this.opts.nextEra == null) {
-        return;
-      }
-
-      let downTotal = 0;
-      let upTotal = 0;
-
-      snapshots.forEach((snapshot) => {
-        downTotal += Number(snapshot.download);
-        upTotal += Number(snapshot.upload);
-      });
-
-      this.opts.nextEra.addData({
-        download: Number(Number(downTotal / snapshots.length).toFixed(1)),
-        upload: Number(Number(upTotal / snapshots.length).toFixed(1)),
-      });
-    });
+  private removeOutdatedData = (): Promise<void> => {
+    const minTimestamp = Date.now() - this.opts.maxTime;
+    return this.db.remove({timestamp: {$lt: minTimestamp}}, {multi: true}).then(
+      () => undefined,
+      () => undefined,
+    );
   };
 
   async addData(data: TransferData): Promise<void> {
-    await this.ready;
-
     const currentTime = Date.now();
 
     if (currentTime - this.lastUpdate >= this.opts.interval - CUMULATIVE_DATA_BUFFER_DIFF) {
@@ -137,31 +74,13 @@ class HistoryEra {
   }
 
   async getData(): Promise<TransferSnapshot[]> {
-    await this.ready;
-
     const minTimestamp = Date.now() - this.opts.maxTime;
 
     return this.db
       .find<TransferSnapshot>({timestamp: {$gte: minTimestamp}})
       .sort({timestamp: 1})
       .then((snapshots) => snapshots.slice(snapshots.length - config.maxHistoryStates));
   }
-
-  async setNextEra(nextEra: HistoryEra): Promise<void> {
-    await this.ready;
-
-    this.opts.nextEra = nextEra;
-
-    let {nextEraUpdateInterval} = this.opts;
-
-    if (nextEraUpdateInterval && nextEraUpdateInterval > MAX_NEXT_ERA_UPDATE_INTERVAL) {
-      nextEraUpdateInterval = MAX_NEXT_ERA_UPDATE_INTERVAL;
-    }
-
-    if (nextEraUpdateInterval) {
-      this.nextEraUpdateInterval = setInterval(this.updateNextEra, nextEraUpdateInterval);
-    }
-  }
 }
 
 export default HistoryEra;
diff --git a/server/routes/api/index.ts b/server/routes/api/index.ts
@@ -7,7 +7,6 @@ import rateLimit from 'express-rate-limit';
 import {contentTokenSchema} from '@shared/schema/api/torrents';
 
 import type {FloodSettings} from '@shared/types/FloodSettings';
-import type {HistorySnapshot} from '@shared/constants/historySnapshotTypes';
 import type {NotificationFetchOptions, NotificationState} from '@shared/types/Notification';
 import type {DirectoryListQuery, DirectoryListResponse, SetFloodSettingsOptions} from '@shared/types/api/index';
 
@@ -163,12 +162,11 @@ router.get<unknown, unknown, unknown, DirectoryListQuery>(
  * @summary Gets transfer history in the given interval
  * @tags Flood
  * @security User
- * @param {HistorySnapshot} snapshot.query - interval
  * @return {TransferHistory} 200 - success response - application/json
  * @return {Error} 500 - failure response - application/json
  */
-router.get<unknown, unknown, unknown, {snapshot: HistorySnapshot}>('/history', (req, res) => {
-  req.services.historyService.getHistory(req.query).then(
+router.get('/history', (req, res) => {
+  req.services.historyService.getHistory().then(
     (snapshot) => {
       res.json(snapshot);
     },