Modified e2e regression tests to reproduce only one bug (#275)

* set 'fork' as the default process start method Signed-off-by: Mark Zhang <bf-109g2@hotmail.com> * Modified e2e regression tests to reproduce only one bug * Fixed test name * Modified e2e regression tests to reproduce only one bug * Fixed test name * Modified unittest to not run the e2e tests --------- Signed-off-by: Mark Zhang <bf-109g2@hotmail.com> Co-authored-by: Mark Zhang <bf-109g2@hotmail.com>
xlab-uiuc · Nov 1, 2023 · 1780f5c · 1780f5c · github-actions · Nov 1, 2023
1 parent b9bc68e
commit 1780f5c
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 6 deletions.
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -1,8 +1,8 @@
 name: End-to-End Test
 on:
   pull_request:
-  schedule:
-    - cron: '0 0 * * 0'  # Run every Sunday at 00:00 UTC
+  # schedule:
+  #   - cron: '0 0 * * 0'  # Run every Sunday at 00:00 UTC
 jobs:
   bug_reproduction:
     timeout-minutes: 1440
@@ -26,4 +26,4 @@ jobs:
           mkdir -m 777 -p profile/data
       - name: Run bug reproduction
         run: |
-          pytest -m "local"
+          pytest -m "singleBugReproduction"
diff --git a/.github/workflows/python-syntax-checker.yml b/.github/workflows/python-syntax-checker.yml
@@ -30,7 +30,7 @@ jobs:
           make
       - name: Run unittest
         run: |
-          pytest -m "not local" --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=acto | tee pytest-coverage.txt
+          pytest -m "not local and not singleBugReproduction" --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=acto | tee pytest-coverage.txt
       - name: Pytest coverage comment
         uses: MishaKav/pytest-coverage-comment@main
         with:

diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml
@@ -19,4 +19,4 @@ jobs:
           make
       - name: Run unittest
         run: |
-          pytest -m "not local"
+          pytest -m "not local and not singleBugReproduction"
diff --git a/pytest.ini b/pytest.ini
@@ -1,3 +1,4 @@
 [pytest]
 markers =
-    local: mark a test to run on a local machine
+    local: mark a test to run on a local machine
+    singleBugReproduction: mark a test to run e2e regressions on a pull request
diff --git a/test/test_bug_reproduction.py b/test/test_bug_reproduction.py
@@ -2,6 +2,7 @@
 import os
 import pathlib
 import queue
+import random
 from typing import Dict, List, Tuple
 import unittest
 
@@ -118,6 +119,7 @@ class TestBugReproduction(unittest.TestCase):
     def __init__(self, methodName: str = "runTest") -> None:
         super().__init__(methodName)
 
+        # TODO: make _num_workers a command line argument
         self._num_workers = 2 # Number of workers to run the test
 
     def test_all_bugs(self):
@@ -146,6 +148,40 @@ def test_all_bugs(self):
 
         self.assertFalse(errors, f'Test failed with {errors}')
 
+@pytest.mark.singleBugReproduction
+class TestSingleBugReproduction(unittest.TestCase):
+
+    def __init__(self, methodName: str = "runTest") -> None:
+        super().__init__(methodName)
+
+        self._num_workers = 1 # Number of workers to run the test
+
+    def test_all_bugs(self):
+        manager = multiprocessing.Manager()
+        workqueue = multiprocessing.Queue()
+
+        operator, bugs = random.choice(list(all_bugs.items()))
+        bug_id, bug_config = random.choice(list(bugs.items()))
+        workqueue.put((operator, bug_id, bug_config))
+
+        reproduction_results: Dict[str, bool] = manager.dict() # workers write reproduction results
+                                                               # to this dict. Bug ID -> if success
+        processes: List[multiprocessing.Process] = []
+        for i in range(self._num_workers):
+            p = multiprocessing.Process(target=run_worker, args=(workqueue, i, reproduction_results))
+            p.start()
+            processes.append(p)
+
+        for p in processes:
+            p.join()
+
+        errors = []
+        for bug_id, if_reproduced in reproduction_results.items():
+            if not if_reproduced:
+                errors.append(bug_id)
+
+        self.assertFalse(errors, f'Test failed with {errors}')
+
 
 if __name__ == '__main__':
     unittest.main()
File	Stmts	Miss	Cover	Missing
acto
__main__.py	86	86	0%	1–174
common.py	354	101	71%	97–98, 102–116, 119–126, 129, 132, 136, 138, 140, 142, 144, 148, 153–162, 203, 236, 249, 291, 315–316, 325, 327, 330, 334–343, 363–366, 415, 418, 432, 460–461, 498–503, 505–507, 514–517, 521, 537–538, 544–555, 614–623, 627, 631, 634–639
deploy.py	143	117	18%	29–32, 35–52, 56, 59–69, 77–107, 110–117, 123–145, 148–161, 173–196, 202–211, 219–237
engine.py	532	486	9%	39–56, 63–79, 91–191, 201–226, 229–285, 301–424, 428–529, 539–564, 568–576, 579–592, 617–733, 736–822, 826–916
oracle_handle.py	24	13	46%	15–18, 26–27, 39–46, 54
reproduce.py	122	81	34%	30–40, 48–52, 58–59, 72–93, 96, 99, 102, 106–108, 111, 114, 122–123, 126, 130, 134, 138, 142–176, 179–189, 192–225
serialization.py	30	16	47%	19–27, 33–39
snapshot.py	62	26	58%	32, 48, 58–79, 82–86, 89
acto/checker
checker.py	13	2	85%	12, 19
checker_set.py	57	12	79%	41–42, 67–78
test_checker.py	162	152	6%	13–246
acto/checker/impl
crash.py	30	2	93%	14, 16
health.py	49	3	94%	36, 65, 81
kubectl_cli.py	29	4	86%	32–33, 44–46
operator_log.py	24	2	92%	24, 32
state.py	225	51	77%	55, 63–64, 69–93, 104–105, 133, 140, 147–149, 166, 265, 288, 317, 319–323, 329–336
state_compare.py	76	3	96%	62, 73, 80
state_condition.py	39	13	67%	18, 24, 29–37, 47, 53
acto/input
get_matched_schemas.py	54	22	59%	12, 47–51, 55–74
input.py	609	489	20%	30–37, 74, 92, 112, 123–126, 129, 159–171, 174–185, 189, 193, 200, 206, 210–389, 410–424, 432–437, 441–454, 463, 479, 482–489, 501, 503, 505–507, 510–523, 528, 536–547, 550–556, 564–591, 595–899, 922–935, 939–986
testcase.py	55	20	64%	40–50, 53, 56, 59, 62–66, 95–96, 100, 103, 109, 116, 119
testplan.py	183	137	25%	14–24, 27–29, 32, 35–42, 45, 48–67, 70–74, 78, 82, 85–91, 99–107, 110–121, 124, 127, 130, 133, 136–138, 141–151, 154–164, 167, 174–185, 188–194, 200, 203–219, 222, 225, 231, 239–244, 247, 250, 253, 259–260, 263–269, 272, 275, 278
value_with_schema.py	337	218	35%	16, 20, 24, 28, 32, 36, 40, 54, 58, 61–67, 70–76, 86–114, 117–124, 128–137, 141–149, 152–156, 159, 162, 166, 180, 183, 186–192, 195–201, 211–230, 233–240, 243, 247–256, 260–272, 275–279, 282, 285, 289, 301, 307, 312, 314, 316, 320, 324, 329–333, 337–340, 349–359, 362–369, 373–376, 380–385, 388–391, 405, 408–412, 416, 421–428, 431–434, 437–439, 442–445, 448–451, 462, 465, 468, 485–498
valuegenerator.py	620	389	37%	20, 24, 28, 32, 43, 47, 51, 55, 76–86, 90–104, 107, 110, 114, 117, 121–130, 133–139, 142, 145, 148, 151, 154, 157–167, 194–199, 202–213, 216, 219, 223, 226–231, 234–237, 240, 243–248, 251–254, 257, 260, 263, 266, 269, 273–282, 285, 288, 291, 294–304, 331–343, 346–347, 350, 353, 357, 360–365, 368–371, 374, 377–382, 385–388, 391, 394, 397, 400, 403, 406, 409–419, 450–482, 485–496, 499–502, 505–508, 512–520, 523, 526, 529, 532, 535, 538–548, 573–589, 592–609, 612, 615, 619–621, 624–628, 631–632, 635–644, 647–653, 656–657, 660–670, 673, 676, 679, 682, 685, 688–698, 710–711, 714–724, 727–730, 733–736, 740, 748, 751–752, 755–765, 768–771, 774–777, 781, 794–797, 800–814, 817, 820, 824, 827–832, 835, 838, 841–846, 849, 852, 855, 858, 861–871, 881, 884, 887, 890, 894, 913, 919, 931, 933, 936–940, 943–946, 951, 955, 957, 961–962
acto/input/known_schemas
base.py	53	13	75%	17–18, 28, 37, 46–47, 56–57, 66, 75, 84–85, 93
cronjob_schemas.py	76	33	57%	13, 16–19, 22, 25, 36–39, 42–47, 50, 53, 59, 62–65, 68, 71, 82, 85–90, 93, 96, 113, 117–119, 131, 137, 140
deployment_schemas.py	59	25	58%	16, 22–27, 30–32, 35, 38, 54–57, 65–67, 70, 78–81, 91, 94
known_schema.py	75	39	48%	28, 31–34, 37, 43, 46–48, 51, 54, 81–84, 102–113, 117–135
pod_disruption_budget_schemas.py	56	22	61%	14–17, 21, 25–27, 30, 41–44, 48, 54, 57, 68–71, 81, 84
pod_schemas.py	797	271	66%	16–19, 23, 28, 32, 40–43, 47, 51–53, 61–64, 68, 73, 83, 92, 151, 156, 160, 167, 171, 178, 182, 238, 242, 247, 251, 294, 298, 303, 307, 335, 338, 341, 344, 347, 350, 353, 356, 359, 362, 365, 368, 393, 397, 400–405, 408, 414, 417, 420, 428, 431, 434, 437, 445, 453, 481, 488–494, 503, 507, 539, 543, 572, 575, 578, 581, 584, 587, 590, 619, 623, 626–631, 634, 642, 645, 648, 664–667, 670–672, 675, 684, 690, 693–696, 699, 702, 713–714, 717–719, 722, 725, 736, 739, 742, 745–748, 752, 756–758, 761–765, 768, 774, 777, 780, 783, 786, 789, 792, 795, 798, 801, 804, 807, 810, 813, 816, 840, 845, 849–857, 860, 866, 869, 872, 875, 878, 881, 884, 887, 890, 893, 896, 899, 902, 905, 908, 929, 933–935, 938–943, 946, 959, 964, 974, 980, 986, 989, 992, 1032, 1036–1038, 1041, 1054, 1059, 1065, 1068–1071, 1074, 1077, 1088–1091, 1094–1096, 1102, 1108, 1111–1114, 1117, 1120, 1133–1136, 1139–1141, 1147, 1153, 1156–1159, 1162, 1165, 1176–1179, 1182–1184, 1190, 1196, 1199–1202, 1205, 1212, 1215–1217, 1223, 1238, 1243, 1247, 1274, 1278, 1291, 1296, 1302, 1305, 1308, 1314, 1317, 1320–1322, 1325, 1342, 1345, 1348, 1374, 1378–1381, 1384, 1402, 1408, 1411, 1414, 1421, 1427, 1469, 1473
resource_schemas.py	149	50	66%	15–19, 22, 25, 28–32, 35, 38, 45, 49, 54, 57–59, 62, 76, 78, 82, 96, 102–105, 108, 111, 121, 134, 147, 154, 159–162, 165, 173–176, 179, 187, 194, 199, 213, 231, 236
service_schemas.py	178	67	62%	13, 16, 19, 25–30, 33, 36, 42, 45–48, 51, 54, 64–67, 70–73, 79, 85, 88–91, 94, 101–102, 105–108, 111, 114, 127, 142, 180, 184, 208, 214, 217, 220, 228–231, 235, 240, 244–246, 249, 257–258, 261, 264, 277–280, 284, 288–290, 293
statefulset_schemas.py	186	61	67%	15–18, 21, 31–36, 42, 49–53, 56–58, 61–63, 66–70, 73–75, 78–80, 83, 90–93, 99–102, 105, 132, 149, 154, 158, 164, 167–170, 173, 176, 190–193, 196–201, 207, 225, 230, 234, 262, 266, 290
storage_schemas.py	179	79	56%	13, 16, 25–30, 33, 36, 42, 48–53, 59, 67–70, 74, 79–80, 83, 89, 92–95, 98, 104–105, 108–111, 114–116, 122, 130–131, 135, 140, 145–148, 154–155, 158, 164, 181–184, 193, 197, 203–205, 211, 214, 228–231, 244, 250–252, 255, 258, 266–269, 273, 277–279, 282
acto/kubectl_client
kubectl.py	23	18	22%	8–14, 23–29, 37–44
acto/kubernetes_engine
base.py	53	30	43%	28, 32, 36, 40, 44, 47–65, 72–86
k3d.py	85	68	20%	17, 21–39, 45, 54–78, 81–97, 100–110, 117–133, 137–139
kind.py	94	76	19%	20–21, 25–56, 62, 71–108, 111–123, 126–139, 146–160
minikube.py	3	3	0%	1–5
acto/monkey_patch
monkey_patch.py	79	24	70%	9, 18, 36, 39, 41, 45–56, 76, 90–95, 106
acto/parse_log
parse_log.py	78	19	76%	71–74, 84–86, 89–91, 96–98, 116–124
acto/post_process
post_diff_test.py	390	231	41%	40, 54–55, 94–95, 166, 176, 180, 185–194, 229, 231, 233–237, 243–260, 268–276, 279–302, 309–318, 321–380, 392–396, 416, 424–454, 457–474, 478–527, 539, 543, 549–550, 552–573, 583–619
post_process.py	104	22	79%	51, 55, 67, 77–88, 100, 103, 138–142, 146, 150, 158, 162
test_post_process.py	28	1	96%	53
acto/runner
runner.py	288	260	10%	24–64, 76–122, 125–134, 137–168, 175–202, 209–232, 235–241, 244–263, 266–271, 282–287, 301–306, 319–395, 400–404, 410–420, 431–454, 459–492
acto/schema
anyof.py	44	20	55%	26, 30–38, 41, 44, 47–49, 52, 55–60
array.py	70	29	59%	31, 43–52, 57, 59–62, 71–78, 81–83, 86–88, 91, 94, 97, 103
base.py	102	57	44%	13–15, 18–20, 23, 26–37, 40, 43, 46–48, 51–61, 64–74, 77, 80–86, 95, 100, 105, 110, 114, 118, 142, 145–149
boolean.py	27	10	63%	16, 20, 23, 26, 29–32, 35, 38
integer.py	26	9	65%	17, 21–23, 26, 29, 32, 35, 38
number.py	30	11	63%	30–32, 35–37, 40, 43, 46, 49, 52
object.py	117	48	59%	44, 46, 51, 66–75, 80, 83, 94–109, 112–120, 123–126, 129, 132, 148, 151–158, 168
oneof.py	44	30	32%	13–19, 22, 25–27, 30–38, 41, 44, 47–49, 52, 55–60
opaque.py	17	6	65%	13, 16, 19, 22, 25, 28
schema.py	42	8	81%	21, 25, 31–34, 49–50
string.py	27	9	67%	25, 29–31, 34, 37, 40, 43, 46
acto/utils
__init__.py	13	1	92%	11
acto_timer.py	31	22	29%	10–15, 19, 22–33, 38–40, 44–47
error_handler.py	43	33	23%	13–30, 38–52, 56–74
k8s_helper.py	64	51	20%	21–27, 39–45, 57–61, 73–79, 83–91, 95–102, 106–127
preprocess.py	69	58	16%	17–73, 93–141, 147–192
process_with_except.py	9	9	0%	1–13
thread_logger.py	15	3	80%	9, 18, 28
TOTAL	8106	4271	47%