Set back to 20 epochs

Author: neekolas

cli/gen.ts

@@ -62,9 +62,9 @@ EXAMPLES:
   yarn gen --help
 
 PRESET COMMANDS:
-  yarn update:local      Update 500 inboxes for local testing
-  yarn update:prod       Update inboxes for production testing
-  yarn restart:prod      Restart production installations (force recreate)
+  yarn gen update:local      Update 500 inboxes for local testing
+  yarn gen update:prod       Update inboxes for production testing
+  yarn gen restart:prod      Restart production installations (force recreate)
 
 For more information, see: cli/readme.md
 `);

forks/README.md

@@ -22,6 +22,9 @@ LOG_LEVEL=debug
 XMTP_ENV=production
 ```
 
+### Running locally
+Before running this suite locally you _must_ run `yarn gen update:local` to pre-populate the database with inboxes to add and remove from the group. Otherwise add/remove member operations will fail, which will not increase the epoch or trigger forks.
+
 ### Fork generation through send testing
 
 The main approach creates intentional conflicts by running parallel operations on shared groups:
@@ -105,8 +108,9 @@ The fork test can inject network chaos (latency, jitter, packet loss) to simulat
 
 **Requirements:**
 - Network chaos requires `--env local`
-- Multinode Docker containers must be running (`./dev/up`)
-- Requires `sudo` access for `tc` and `iptables` commands
+- Multinode Docker containers must be running (`./multinode/up`)
+- Must be run on linux with `tc` and `iptables` commands available. Will not work on MacOS.
+- Requires `sudo` access
 
 **Chaos Levels:**
 
@@ -145,11 +149,3 @@ chaosLevel: high
   packetLoss: 0-5%
   interval: 10000ms
 ```
-
-### Log processing features
-
-- **Clean slate**: Removes old logs and data before starting
-- **Continuous capture**: Each iteration captures debug logs
-- **ANSI cleaning**: Strips escape codes for analysis
-- **Fork counting**: Automatically counts detected conflicts
-- **Graceful interruption**: Ctrl+C exits cleanly

forks/cli.ts

@@ -8,7 +8,6 @@ import {
   epochRotationOperations,
   groupCount,
   installationCount,
-  network,
   NODE_VERSION,
   otherOperations,
   parallelOperations,
@@ -89,8 +88,8 @@ function runForkTest(options: ForkOptions): void {
         CHAOS_LEVEL: options.chaosLevel,
       },
     });
-  } catch {
-    console.log("Error running fork test");
+  } catch (e) {
+    console.error("Error running fork test", e);
     // Test may fail if forks are detected, that's expected
     // We'll analyze the logs afterward
   }
@@ -111,7 +110,7 @@ function logForkMatrixParameters(options: ForkOptions): void {
   );
   console.info(`otherOperations: ${JSON.stringify(otherOperations)}`);
   console.info(`targetEpoch: ${targetEpoch}`);
-  console.info(`network: ${network || "undefined"}`);
+  console.info(`network: ${options.env || "undefined"}`);
   console.info(`randomInboxIdsCount: ${randomInboxIdsCount}`);
   console.info(`installationCount: ${installationCount}`);
   console.info(`testName: ${testName}`);
@@ -192,15 +191,6 @@ async function runForkDetection(options: ForkOptions): Promise<void> {
       stats.runsWithoutForks++;
       console.info(`Run ${i}/${options.count}: ⚪ No forks`);
     }
-
-    // Clean up empty cleaned directory if it exists
-    const logsDir = path.join(process.cwd(), "logs", "cleaned");
-    if (fs.existsSync(logsDir)) {
-      const files = fs.readdirSync(logsDir);
-      if (files.length === 0) {
-        fs.rmdirSync(logsDir);
-      }
-    }
   }
 
   // Display final statistics

forks/config.ts

@@ -21,10 +21,10 @@ export const epochRotationOperations = {
   removeMember: true, // removes a random member from the group
 };
 export const otherOperations = {
-  createInstallation: true, // creates a new installation for a random worker
+  createInstallation: false, // creates a new installation for a random worker
   sendMessage: true, // sends a message to the group
 };
-export const targetEpoch = 150n; // The target epoch to stop the test (epochs are when performing forks to the group)
+export const targetEpoch = 30n; // The target epoch to stop the test (epochs are when performing forks to the group)
 export const network = process.env.XMTP_ENV; // Network environment setting
 export const randomInboxIdsCount = 10; // How many inboxIds to use randomly in the add/remove operations
 export const installationCount = 2; // How many installations to use randomly in the createInstallation operations
@@ -63,12 +63,12 @@ export const chaosPresets: Record<ChaosLevel, ChaosPreset> = {
     interval: 10000, // 10 seconds
   },
   high: {
-    delayMin: 100,
+    delayMin: 0,
     delayMax: 500,
-    jitterMin: 0,
-    jitterMax: 100,
+    jitterMin: 50,
+    jitterMax: 200,
     lossMin: 0,
-    lossMax: 5,
+    lossMax: 10,
     interval: 10000, // 10 seconds
   },
 };
@@ -90,4 +90,6 @@ export const multinodeContainers = [
   "multinode-node2-1",
   "multinode-node3-1",
   "multinode-node4-1",
+  // Include the MLS validation service to add some additional chaos
+  "multinode-validation-1",
 ];

forks/constants.ts

@@ -0,0 +1 @@
+export const forkDetectedString = "[FORK DETECTED]";

forks/forks.test.ts

@@ -20,8 +20,61 @@ import {
   targetEpoch,
   testName,
   workerNames,
+  type ChaosPreset,
 } from "./config";
 
+const startChaos = (
+  allNodes: DockerContainer[],
+  preset: ChaosPreset,
+): NodeJS.Timeout => {
+  console.log(`[chaos] Initialized ${allNodes.length} Docker containers`);
+
+  // Validate containers are running
+  for (const node of allNodes) {
+    try {
+      // Test if container exists by trying to get its IP
+      if (!node.ip) {
+        throw new Error(`Container ${node.name} has no IP address`);
+      }
+    } catch {
+      throw new Error(
+        `Docker container ${node.name} is not running. Network chaos requires local multinode setup (./dev/up).`,
+      );
+    }
+  }
+  console.log("[chaos] All Docker containers validated");
+
+  // Function to apply chaos to all nodes
+  const applyChaos = () => {
+    console.log(
+      "[chaos] Applying jitter, delay, and drop rules to all nodes...",
+    );
+    for (const node of allNodes) {
+      const delay = Math.floor(
+        preset.delayMin + Math.random() * (preset.delayMax - preset.delayMin),
+      );
+      const jitter = Math.floor(
+        preset.jitterMin +
+          Math.random() * (preset.jitterMax - preset.jitterMin),
+      );
+      const loss =
+        preset.lossMin + Math.random() * (preset.lossMax - preset.lossMin);
+
+      try {
+        node.addJitter(delay, jitter);
+        if (Math.random() < 0.5) node.addLoss(loss);
+      } catch (err) {
+        console.warn(`[chaos] Error applying netem on ${node.name}:`, err);
+      }
+    }
+  };
+
+  // Apply chaos immediately
+  applyChaos();
+
+  return setInterval(applyChaos, preset.interval);
+};
+
 describe(testName, () => {
   setupDurationTracking({ testName });
 
@@ -91,79 +144,29 @@ describe(testName, () => {
 
         // Initialize Docker containers for multinode setup
         allNodes = multinodeContainers.map((name) => new DockerContainer(name));
-        console.log(`[chaos] Initialized ${allNodes.length} Docker containers`);
-
-        // Validate containers are running
-        for (const node of allNodes) {
-          try {
-            // Test if container exists by trying to get its IP
-            if (!node.ip) {
-              throw new Error(`Container ${node.name} has no IP address`);
-            }
-          } catch (_err) {
-            throw new Error(
-              `Docker container ${node.name} is not running. Network chaos requires local multinode setup (./dev/up).`,
-            );
-          }
-        }
-        console.log("[chaos] All Docker containers validated");
-
         const preset = chaosPresets[chaosConfig.level];
-
-        // Function to apply chaos to all nodes
-        const applyChaos = () => {
-          console.log(
-            "[chaos] Applying jitter, delay, and drop rules to all nodes...",
-          );
-          for (const node of allNodes) {
-            const delay = Math.floor(
-              preset.delayMin +
-                Math.random() * (preset.delayMax - preset.delayMin),
-            );
-            const jitter = Math.floor(
-              preset.jitterMin +
-                Math.random() * (preset.jitterMax - preset.jitterMin),
-            );
-            const loss =
-              preset.lossMin +
-              Math.random() * (preset.lossMax - preset.lossMin);
-
-            try {
-              node.addJitter(delay, jitter);
-              if (Math.random() < 0.5) node.addLoss(loss);
-            } catch (err) {
-              console.warn(
-                `[chaos] Error applying netem on ${node.name}:`,
-                err,
-              );
-            }
-          }
-        };
-
-        // Apply chaos immediately
-        applyChaos();
-
         // Then set interval for continued chaos
-        chaosInterval = setInterval(applyChaos, preset.interval);
+        chaosInterval = startChaos(allNodes, preset);
         console.log(`[chaos] Started chaos interval (${preset.interval}ms)`);
+      }
 
-        // Start periodic verification during chaos
-        const verifyLoop = () => {
-          verifyInterval = setInterval(() => {
-            void (async () => {
-              try {
-                console.log("[verify] Checking forks under chaos");
-                await workers.checkForks();
-              } catch (e) {
-                console.warn("[verify] Skipping check due to exception:", e);
-              }
-            })();
-          }, 10 * 1000);
-        };
+      // Start periodic verification during chaos
+      const verifyLoop = () => {
+        verifyInterval = setInterval(() => {
+          void (async () => {
+            try {
+              console.log("[verify] Checking forks under chaos");
+              await workers.checkForks();
+            } catch (e) {
+              console.warn("[verify] Skipping check due to exception:", e);
+              throw e;
+            }
+          })();
+        }, 10 * 1000);
+      };
 
-        verifyLoop();
-        console.log("[chaos] Started verification interval (10000ms)");
-      }
+      verifyLoop();
+      console.log("Started verification interval (10000ms)");
 
       // Create groups
       const groupOperationPromises = Array.from(
@@ -228,7 +231,13 @@ describe(testName, () => {
       );
 
       await Promise.all(groupOperationPromises);
+      await workers.checkForks();
+    } catch (e) {
+      console.error("Error during fork testing:", e);
     } finally {
+      if (verifyInterval) {
+        clearInterval(verifyInterval);
+      }
       // Clean up chaos if it was enabled
       if (chaosConfig.enabled) {
         console.log("[chaos] Cleaning up network chaos...");
@@ -237,9 +246,6 @@ describe(testName, () => {
         if (chaosInterval) {
           clearInterval(chaosInterval);
         }
-        if (verifyInterval) {
-          clearInterval(verifyInterval);
-        }
 
         // Clear network rules
         for (const node of allNodes) {
@@ -253,10 +259,6 @@ describe(testName, () => {
           }
         }
 
-        // Cooldown period to allow in-flight messages to be processed
-        console.log("[chaos] Waiting 5s cooldown before final validation");
-        await new Promise((r) => setTimeout(r, 5000));
-
         console.log("[chaos] Cleanup complete");
       }
     }

helpers/analyzer.ts

@@ -1,5 +1,6 @@
 import fs from "fs";
 import path from "path";
+import { forkDetectedString } from "forks/constants";
 import { processLogFile, stripAnsi } from "./logger";
 
 // Known test issues for tracking
@@ -180,11 +181,6 @@ export async function cleanForksLogs(
   const logsDir = path.join(process.cwd(), "logs");
   const outputDir = path.join(logsDir, "cleaned");
 
-  if (!fs.existsSync(logsDir)) {
-    console.debug("No logs directory found");
-    return;
-  }
-
   if (!fs.existsSync(outputDir)) {
     await fs.promises.mkdir(outputDir, { recursive: true });
   }
@@ -214,7 +210,7 @@ export async function cleanForksLogs(
       // Check if the file contains fork-related content
       const containsForkContent = await fileContainsString(
         rawFilePath,
-        "may have forked",
+        forkDetectedString,
       );
 
       // Always preserve raw logs for debugging/analysis

helpers/logger.ts

@@ -2,6 +2,7 @@ import fs from "fs";
 import path from "path";
 import winston from "winston";
 import "dotenv/config";
+import { forkDetectedString } from "forks/constants";
 
 // Consolidated ANSI escape code regex
 // eslint-disable-next-line no-control-regex
@@ -44,7 +45,7 @@ export async function processLogFile(
 
     let buffer = "";
     let foundForkLine = false;
-    const targetString = "may be fork";
+    const targetString = forkDetectedString;
 
     readStream.on("data", (chunk: string | Buffer) => {
       if (foundForkLine) {