feat: prune invalid SSZ objects (#5388)

* Prune old invalid SSZ files

* Fix pruneOldFilesInDir

* Capitalize error message

* Rephrase CLI flag description

* Check mtimeMs instead of ctimeMs

* Add tests for pruneOldFilesInDir

---------

Co-authored-by: dapplion <35266934+dapplion@users.noreply.github.com>
Co-authored-by: Nico Flaig <nflaig@protonmail.com>

packages/cli/src/cmds/beacon/handler.ts

@@ -11,13 +11,23 @@ import {LoggerNode, getNodeLogger} from "@lodestar/logger/node";
 import {GlobalArgs, parseBeaconNodeArgs} from "../../options/index.js";
 import {BeaconNodeOptions, getBeaconConfigFromArgs} from "../../config/index.js";
 import {getNetworkBootnodes, getNetworkData, isKnownNetworkName, readBootnodes} from "../../networks/index.js";
-import {onGracefulShutdown, mkdir, writeFile600Perm, cleanOldLogFiles, parseLoggerArgs} from "../../util/index.js";
+import {
+  onGracefulShutdown,
+  mkdir,
+  writeFile600Perm,
+  cleanOldLogFiles,
+  parseLoggerArgs,
+  pruneOldFilesInDir,
+} from "../../util/index.js";
 import {getVersionData} from "../../util/version.js";
 import {BeaconArgs} from "./options.js";
 import {getBeaconPaths} from "./paths.js";
 import {initBeaconState} from "./initBeaconState.js";
 import {initPeerIdAndEnr} from "./initPeerIdAndEnr.js";
 
+const DEFAULT_RETENTION_SSZ_OBJECTS_HOURS = 15 * 24;
+const HOURS_TO_MS = 3600 * 1000;
+
 /**
  * Runs a beacon node.
  */
@@ -80,8 +90,28 @@ export async function beaconHandler(args: BeaconArgs & GlobalArgs): Promise<void
       metricsRegistries,
     });
 
-    if (args.attachToGlobalThis) (globalThis as unknown as {bn: BeaconNode}).bn = node;
+    // dev debug option to have access to the BN instance
+    if (args.attachToGlobalThis) {
+      (globalThis as unknown as {bn: BeaconNode}).bn = node;
+    }
 
+    // Prune invalid SSZ objects every interval
+    const {persistInvalidSszObjectsDir} = args;
+    const pruneInvalidSSZObjectsInterval = persistInvalidSszObjectsDir
+      ? setInterval(() => {
+          try {
+            pruneOldFilesInDir(
+              persistInvalidSszObjectsDir,
+              (args.persistInvalidSszObjectsRetentionHours ?? DEFAULT_RETENTION_SSZ_OBJECTS_HOURS) * HOURS_TO_MS
+            );
+          } catch (e) {
+            logger.warn("Error pruning invalid SSZ objects", {persistInvalidSszObjectsDir}, e as Error);
+          }
+          // Run every ~1 hour
+        }, HOURS_TO_MS)
+      : null;
+
+    // Intercept SIGINT signal, to perform final ops before exiting
     onGracefulShutdown(async () => {
       if (args.persistNetworkIdentity) {
         try {
@@ -93,6 +123,10 @@ export async function beaconHandler(args: BeaconArgs & GlobalArgs): Promise<void
         }
       }
       abortController.abort();
+
+      if (pruneInvalidSSZObjectsInterval !== null) {
+        clearInterval(pruneInvalidSSZObjectsInterval);
+      }
     }, logger.info.bind(logger));
 
     abortController.signal.addEventListener(

packages/cli/src/cmds/beacon/options.ts

@@ -15,6 +15,7 @@ type BeaconExtraArgs = {
   beaconDir?: string;
   dbDir?: string;
   persistInvalidSszObjectsDir?: string;
+  persistInvalidSszObjectsRetentionHours?: number;
   peerStoreDir?: string;
   persistNetworkIdentity?: boolean;
   private?: boolean;
@@ -86,6 +87,12 @@ export const beaconExtraOptions: CliCommandOptions<BeaconExtraArgs> = {
     type: "string",
   },
 
+  persistInvalidSszObjectsRetentionHours: {
+    description: "Number of hours to keep invalid SSZ objects on local disk",
+    hidden: true,
+    type: "number",
+  },
+
   peerStoreDir: {
     hidden: true,
     description: "Peer store directory",

packages/cli/src/util/index.ts

@@ -10,6 +10,7 @@ export * from "./logger.js";
 export * from "./object.js";
 export * from "./passphrase.js";
 export * from "./process.js";
+export * from "./pruneOldFilesInDir.js";
 export * from "./sleep.js";
 export * from "./stripOffNewlines.js";
 export * from "./types.js";

packages/cli/src/util/pruneOldFilesInDir.ts

@@ -0,0 +1,17 @@
+import fs from "node:fs";
+import path from "node:path";
+
+export function pruneOldFilesInDir(dirpath: string, maxAgeMs: number): void {
+  for (const entryName of fs.readdirSync(dirpath)) {
+    const entryPath = path.join(dirpath, entryName);
+
+    const stat = fs.statSync(entryPath);
+    if (stat.isDirectory()) {
+      pruneOldFilesInDir(entryPath, maxAgeMs);
+    } else if (stat.isFile()) {
+      if (Date.now() - stat.mtimeMs > maxAgeMs) {
+        fs.unlinkSync(entryPath);
+      }
+    }
+  }
+}

packages/cli/test/unit/util/pruneOldFilesInDir.test.ts

@@ -0,0 +1,67 @@
+import fs from "node:fs";
+import path from "node:path";
+import {rimraf} from "rimraf";
+import {expect} from "chai";
+import {pruneOldFilesInDir} from "../../../src/util/index.js";
+import {testFilesDir} from "../../utils.js";
+
+describe("pruneOldFilesInDir", () => {
+  const DAYS_TO_MS = 24 * 60 * 60 * 1000;
+  const dataDir = path.join(testFilesDir, "prune-old-files-test");
+  const newFile = "newFile.txt";
+  const oldFile = "oldFile.txt";
+
+  beforeEach(() => {
+    fs.mkdirSync(dataDir, {recursive: true});
+    createFileWithAge(path.join(dataDir, oldFile), 2);
+    createFileWithAge(path.join(dataDir, newFile), 0);
+  });
+
+  afterEach(() => {
+    rimraf.sync(dataDir);
+  });
+
+  it("should delete old files", () => {
+    pruneOldFilesInDir(dataDir, DAYS_TO_MS);
+
+    const files = fs.readdirSync(dataDir);
+    expect(files).to.not.include(oldFile);
+  });
+
+  it("should not delete new files", () => {
+    pruneOldFilesInDir(dataDir, DAYS_TO_MS);
+
+    const files = fs.readdirSync(dataDir);
+    expect(files).to.include(newFile);
+  });
+
+  it("should delete old files in nested directories", () => {
+    const nestedDir = path.join(dataDir, "prune-old-files-nested");
+
+    fs.mkdirSync(nestedDir);
+    createFileWithAge(path.join(nestedDir, "nestedFile.txt"), 2);
+
+    pruneOldFilesInDir(dataDir, DAYS_TO_MS);
+
+    expect(fs.readdirSync(nestedDir)).to.be.empty;
+  });
+
+  it("should handle empty directories", () => {
+    const emptyDir = path.join(dataDir, "prune-old-files-empty");
+    fs.mkdirSync(emptyDir, {recursive: true});
+
+    pruneOldFilesInDir(emptyDir, DAYS_TO_MS);
+
+    expect(fs.readdirSync(emptyDir)).to.be.empty;
+  });
+
+  function createFileWithAge(path: string, ageInDays: number): void {
+    // Create a new empty file
+    fs.closeSync(fs.openSync(path, "w"));
+
+    if (ageInDays > 0) {
+      // Alter file's access and modification dates
+      fs.utimesSync(path, Date.now() / 1000, (Date.now() - ageInDays * DAYS_TO_MS) / 1000);
+    }
+  }
+});