Modified e2e regression tests to reproduce only one bug

.github/workflows/e2e-test.yml

@@ -1,8 +1,8 @@
 name: End-to-End Test
 on:
   pull_request:
-  schedule:
-    - cron: '0 0 * * 0'  # Run every Sunday at 00:00 UTC
+  # schedule:
+  #   - cron: '0 0 * * 0'  # Run every Sunday at 00:00 UTC
 jobs:
   bug_reproduction:
     timeout-minutes: 1440
@@ -26,4 +26,4 @@ jobs:
           mkdir -m 777 -p profile/data
       - name: Run bug reproduction
         run: |
-          pytest -m "local"
+          pytest -m "singleBugReproduction"

.github/workflows/python-syntax-checker.yml

@@ -30,7 +30,7 @@ jobs:
           make
       - name: Run unittest
         run: |
-          pytest -m "not local" --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=acto | tee pytest-coverage.txt
+          pytest -m "not local and not singleBugReproduction" --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=acto | tee pytest-coverage.txt
       - name: Pytest coverage comment
         uses: MishaKav/pytest-coverage-comment@main
         with:

.github/workflows/unittest.yaml

@@ -19,4 +19,4 @@ jobs:
           make
       - name: Run unittest
         run: |
-          pytest -m "not local"
+          pytest -m "not local and not singleBugReproduction"

acto/runner/runner.py

@@ -1,7 +1,7 @@
 import base64
 import queue
 import time
-from multiprocessing import Process, Queue
+from multiprocessing import Process, Queue, set_start_method, get_start_method
 
 import yaml
 
@@ -60,6 +60,9 @@ def __init__(self,
             'role_binding': self.rbacAuthorizationV1Api.list_namespaced_role_binding,
         }
 
+        if get_start_method() != "fork":
+            set_start_method("fork")
+
     def run(self, input: dict, generation: int) -> Tuple[Snapshot, bool]:
         '''Simply run the cmd and dumps system_state, delta, operator log, events and input files without checking. 
            The function blocks until system converges.

pytest.ini

@@ -1,3 +1,4 @@
 [pytest]
 markers =
-    local: mark a test to run on a local machine
+    local: mark a test to run on a local machine
+    singleBugReproduction: mark a test to run e2e regressions on a pull request

test/test_bug_reproduction.py

@@ -2,6 +2,7 @@
 import os
 import pathlib
 import queue
+import random
 from typing import Dict, List, Tuple
 import unittest
 
@@ -118,6 +119,7 @@ class TestBugReproduction(unittest.TestCase):
     def __init__(self, methodName: str = "runTest") -> None:
         super().__init__(methodName)
 
+        # TODO: make _num_workers a command line argument
         self._num_workers = 2 # Number of workers to run the test
 
     def test_all_bugs(self):
@@ -146,6 +148,40 @@ def test_all_bugs(self):
 
         self.assertFalse(errors, f'Test failed with {errors}')
 
+@pytest.mark.singleBugReproduction
+class TestSingleBugReproduction(unittest.TestCase):
+
+    def __init__(self, methodName: str = "runTest") -> None:
+        super().__init__(methodName)
+
+        self._num_workers = 1 # Number of workers to run the test
+
+    def test_all_bugs(self):
+        manager = multiprocessing.Manager()
+        workqueue = multiprocessing.Queue()
+
+        operator, bugs = random.choice(list(all_bugs.items()))
+        bug_id, bug_config = random.choice(list(bugs.items()))
+        workqueue.put((operator, bug_id, bug_config))
+
+        reproduction_results: Dict[str, bool] = manager.dict() # workers write reproduction results
+                                                               # to this dict. Bug ID -> if success
+        processes: List[multiprocessing.Process] = []
+        for i in range(self._num_workers):
+            p = multiprocessing.Process(target=run_worker, args=(workqueue, i, reproduction_results))
+            p.start()
+            processes.append(p)
+
+        for p in processes:
+            p.join()
+
+        errors = []
+        for bug_id, if_reproduced in reproduction_results.items():
+            if not if_reproduced:
+                errors.append(bug_id)
+
+        self.assertFalse(errors, f'Test failed with {errors}')
+
 
 if __name__ == '__main__':
     unittest.main()