Add network chaos to fork tests

Author: neekolas

forks/README.md

@@ -99,6 +99,53 @@ The CLI provides statistics including:
 - Fork detection rate
 - Average forks per run
 
+### Network Chaos Testing
+
+The fork test can inject network chaos (latency, jitter, packet loss) to simulate adverse network conditions. This helps identify forks that occur under realistic network stress.
+
+**Requirements:**
+- Network chaos requires `--env local`
+- Multinode Docker containers must be running (`./dev/up`)
+- Requires `sudo` access for `tc` and `iptables` commands
+
+**Chaos Levels:**
+
+| Level  | Delay Range | Jitter Range | Packet Loss | Interval |
+|--------|-------------|--------------|-------------|----------|
+| low    | 50-150ms    | 0-50ms       | 0-2%        | 15s      |
+| medium | 100-300ms   | 0-75ms       | 0-3.5%      | 10s      |
+| high   | 100-500ms   | 0-100ms      | 0-5%        | 10s      |
+
+**Usage:**
+
+```bash
+# Run with default (medium) chaos
+yarn fork --env local --chaos-enabled
+
+# Run with high chaos level
+yarn fork --env local --chaos-enabled --chaos-level high
+
+# Run 50 iterations with low chaos
+yarn fork --count 50 --env local --chaos-enabled --chaos-level low
+```
+
+**How it works:**
+1. Initializes Docker container handles for all multinode nodes
+2. Applies random network conditions (within preset ranges) at regular intervals
+3. Runs the fork test as normal while chaos is active
+4. Cleans up network rules when test completes (even if test fails)
+
+**Example output:**
+```
+NETWORK CHAOS PARAMETERS
+chaosEnabled: true
+chaosLevel: high
+  delay: 100-500ms
+  jitter: 0-100ms
+  packetLoss: 0-5%
+  interval: 10000ms
+```
+
 ### Log processing features
 
 - **Clean slate**: Removes old logs and data before starting

forks/cli.ts

@@ -4,6 +4,7 @@ import path from "path";
 import { cleanAllRawLogs, cleanForksLogs } from "@helpers/analyzer";
 import "dotenv/config";
 import {
+  chaosPresets,
   epochRotationOperations,
   groupCount,
   installationCount,
@@ -15,13 +16,16 @@ import {
   targetEpoch,
   testName,
   workerNames,
+  type ChaosLevel,
 } from "./config";
 
 interface ForkOptions {
   count: number; // Number of times to run the process (default: 100)
   cleanAll: boolean; // Clean all raw logs before starting
   removeNonMatching: boolean; // Remove non-matching logs
   env?: string; // XMTP environment (local, dev, production)
+  chaosEnabled: boolean; // Enable network chaos
+  chaosLevel: ChaosLevel; // Chaos level (low, medium, high)
 }
 
 function showHelp() {
@@ -37,13 +41,22 @@ OPTIONS:
   --remove-non-matching      Remove logs that don't contain fork content [default: true]
   --no-remove-non-matching   Keep logs that don't contain fork content
   --env <environment>        XMTP environment (local, dev, production) [default: dev]
+  --chaos-enabled            Enable network chaos testing (requires --env local)
+  --chaos-level <level>      Chaos level: low, medium, high [default: medium]
   -h, --help                 Show this help message
 
+CHAOS LEVELS:
+  low      - Delay: 50-150ms, Jitter: 0-50ms, Loss: 0-2%, Interval: 15s
+  medium   - Delay: 100-300ms, Jitter: 0-75ms, Loss: 0-3.5%, Interval: 10s
+  high     - Delay: 100-500ms, Jitter: 0-100ms, Loss: 0-5%, Interval: 10s
+
 EXAMPLES:
-  yarn fork                    # Run 100 times and get stats
-  yarn fork --count 50          # Run 50 times
-  yarn fork --clean-all         # Clean all raw logs before starting
-  yarn fork --count 200 --env local # Run 200 times on local environment
+  yarn fork                              # Run 100 times and get stats
+  yarn fork --count 50                   # Run 50 times
+  yarn fork --clean-all                  # Clean all raw logs before starting
+  yarn fork --count 200 --env local      # Run 200 times on local environment
+  yarn fork --env local --chaos-enabled  # Run with medium network chaos
+  yarn fork --env local --chaos-enabled --chaos-level high  # Run with high chaos
 
 For more information, see: forks/README.md
 `);
@@ -63,16 +76,21 @@ function getForkCount(): number {
 /**
  * Run the fork test (suppress output)
  */
-function runForkTest(env?: string): void {
-  const envFlag = env ? `--env ${env}` : "";
+function runForkTest(options: ForkOptions): void {
+  const envFlag = options.env ? `--env ${options.env}` : "";
   const command = `yarn test forks ${envFlag} --log warn --file`.trim();
 
   try {
     execSync(command, {
       stdio: "ignore",
-      env: { ...process.env },
+      env: {
+        ...process.env,
+        CHAOS_ENABLED: options.chaosEnabled ? "true" : "false",
+        CHAOS_LEVEL: options.chaosLevel,
+      },
     });
   } catch {
+    console.log("Error running fork test");
     // Test may fail if forks are detected, that's expected
     // We'll analyze the logs afterward
   }
@@ -81,7 +99,7 @@ function runForkTest(env?: string): void {
 /**
  * Log fork matrix parameters from shared config
  */
-function logForkMatrixParameters(): void {
+function logForkMatrixParameters(options: ForkOptions): void {
   console.info("\nFORK MATRIX PARAMETERS");
   console.info("-".repeat(60));
   console.info(`groupCount: ${groupCount}`);
@@ -97,6 +115,18 @@ function logForkMatrixParameters(): void {
   console.info(`randomInboxIdsCount: ${randomInboxIdsCount}`);
   console.info(`installationCount: ${installationCount}`);
   console.info(`testName: ${testName}`);
+
+  if (options.chaosEnabled) {
+    const preset = chaosPresets[options.chaosLevel];
+    console.info("\nNETWORK CHAOS PARAMETERS");
+    console.info(`chaosEnabled: true`);
+    console.info(`chaosLevel: ${options.chaosLevel}`);
+    console.info(`  delay: ${preset.delayMin}-${preset.delayMax}ms`);
+    console.info(`  jitter: ${preset.jitterMin}-${preset.jitterMax}ms`);
+    console.info(`  packetLoss: ${preset.lossMin}-${preset.lossMax}%`);
+    console.info(`  interval: ${preset.interval}ms`);
+  }
+
   console.info("-".repeat(60) + "\n");
 }
 
@@ -108,8 +138,17 @@ async function runForkDetection(options: ForkOptions): Promise<void> {
   console.info("XMTP Fork Detection CLI");
   console.info("=".repeat(60));
 
+  // Validate chaos requirements
+  if (options.chaosEnabled && options.env !== "local") {
+    console.error("\n❌ Error: Network chaos testing requires --env local");
+    console.error(
+      "Network chaos manipulates Docker containers which are only available in local environment.\n",
+    );
+    process.exit(1);
+  }
+
   // Log fork matrix parameters once
-  logForkMatrixParameters();
+  logForkMatrixParameters(options);
 
   console.info(`Running fork detection process ${options.count} time(s)...\n`);
 
@@ -130,7 +169,7 @@ async function runForkDetection(options: ForkOptions): Promise<void> {
   // Run the test N times
   for (let i = 1; i <= options.count; i++) {
     // Run the fork test (silently)
-    runForkTest(options.env);
+    runForkTest(options);
 
     // Clean and analyze fork logs after the test (suppress output)
     const originalConsoleDebug = console.debug;
@@ -211,6 +250,8 @@ async function main() {
     cleanAll: false,
     removeNonMatching: true,
     env: process.env.XMTP_ENV || "dev",
+    chaosEnabled: false,
+    chaosLevel: "medium",
   };
 
   // Parse arguments
@@ -260,6 +301,25 @@ async function main() {
           process.exit(1);
         }
         break;
+      case "--chaos-enabled":
+        options.chaosEnabled = true;
+        break;
+      case "--chaos-level":
+        if (i + 1 < args.length) {
+          const level = args[i + 1] as ChaosLevel;
+          if (!["low", "medium", "high"].includes(level)) {
+            console.error("--chaos-level must be one of: low, medium, high");
+            process.exit(1);
+          }
+          options.chaosLevel = level;
+          i++; // Skip next argument
+        } else {
+          console.error(
+            "--chaos-level flag requires a value (e.g., --chaos-level medium)",
+          );
+          process.exit(1);
+        }
+        break;
       default:
         console.error(`Unknown option: ${arg}`);
         console.error("Use --help for usage information");

forks/config.ts

@@ -24,8 +24,70 @@ export const otherOperations = {
   createInstallation: true, // creates a new installation for a random worker
   sendMessage: true, // sends a message to the group
 };
-export const targetEpoch = 20n; // The target epoch to stop the test (epochs are when performing forks to the group)
+export const targetEpoch = 150n; // The target epoch to stop the test (epochs are when performing forks to the group)
 export const network = process.env.XMTP_ENV; // Network environment setting
 export const randomInboxIdsCount = 10; // How many inboxIds to use randomly in the add/remove operations
 export const installationCount = 2; // How many installations to use randomly in the createInstallation operations
 export const testName = "forks";
+
+// Network chaos configuration
+export type ChaosLevel = "low" | "medium" | "high";
+
+export interface ChaosPreset {
+  delayMin: number; // Minimum delay in ms
+  delayMax: number; // Maximum delay in ms
+  jitterMin: number; // Minimum jitter in ms
+  jitterMax: number; // Maximum jitter in ms
+  lossMin: number; // Minimum packet loss percentage (0-100)
+  lossMax: number; // Maximum packet loss percentage (0-100)
+  interval: number; // How often to apply chaos in ms
+}
+
+export const chaosPresets: Record<ChaosLevel, ChaosPreset> = {
+  low: {
+    delayMin: 50,
+    delayMax: 150,
+    jitterMin: 0,
+    jitterMax: 50,
+    lossMin: 0,
+    lossMax: 2,
+    interval: 15000, // 15 seconds
+  },
+  medium: {
+    delayMin: 100,
+    delayMax: 300,
+    jitterMin: 0,
+    jitterMax: 75,
+    lossMin: 0,
+    lossMax: 3.5,
+    interval: 10000, // 10 seconds
+  },
+  high: {
+    delayMin: 100,
+    delayMax: 500,
+    jitterMin: 0,
+    jitterMax: 100,
+    lossMin: 0,
+    lossMax: 5,
+    interval: 10000, // 10 seconds
+  },
+};
+
+export interface ChaosConfig {
+  enabled: boolean;
+  level: ChaosLevel;
+}
+
+// Parse chaos config from environment
+export const chaosConfig: ChaosConfig = {
+  enabled: process.env.CHAOS_ENABLED === "true",
+  level: (process.env.CHAOS_LEVEL as ChaosLevel) || "medium",
+};
+
+// Multinode container names for local environment chaos testing
+export const multinodeContainers = [
+  "multinode-node1-1",
+  "multinode-node2-1",
+  "multinode-node3-1",
+  "multinode-node4-1",
+];