Refactor unittest and integration-tests and CI

Separate unittest and integration-tests into separate directories for easier management.
Change CI to collect unittest and integration-tests' coverage, merge them, and report together.

Signed-off-by: Tyler Gu <jiaweig3@illinois.edu>

.github/workflows/unittest.yaml

@@ -1,13 +1,36 @@
 name: Regression Test
-on: [push]
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
 jobs:
+  check_should_skip:
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@v5.3.1
+        with:
+          skip_after_successful_duplicate: 'true'
   unittest:
     runs-on: ubuntu-latest
+    needs: check_should_skip
+    if: ${{ needs.check_should_skip.outputs.should_skip != 'true' }}
+    permissions:
+      # Gives the action the necessary permissions for publishing new
+      # comments in pull requests.
+      pull-requests: write
+      # Gives the action the necessary permissions for pushing data to the
+      # python-coverage-comment-action branch, and for editing existing
+      # comments (to avoid publishing multiple comments in the same PR)
+      contents: write
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - name: Install Packages
         run: pip install codespell flake8 
       - name: Check Spelling
@@ -23,11 +46,79 @@ jobs:
           pip install -r requirements.txt
           make
       - name: Run unittest
+        run: >
+          COVERAGE_FILE=.coverage.${{ github.sha }}.unittest
+          pytest acto
+          --junitxml=pytest.xml
+          --cov-report=
+          --cov=acto
+      - name: Upload coverage
+        uses: actions/upload-artifact@v4
+        with:
+          name: .coverage.${{ github.sha }}.unittest
+          path: .coverage.${{ github.sha }}.unittest
+          retention-days: 1
+  integration-test:
+    runs-on: ubuntu-latest
+    needs: check_should_skip
+    if: ${{ needs.check_should_skip.outputs.should_skip != 'true' }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install Packages
+        run: pip install codespell flake8 
+      - name: Check Spelling
+        run: codespell --skip="./data"|| true  
+      - name: Check Syntax Error
+        run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.20.5'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          make
+      - name: Run integration test
+        run: >
+          COVERAGE_FILE=.coverage.${{ github.sha }}.integration-test
+          pytest test/integration_tests
+          --junitxml=pytest.xml
+          --cov-report=
+          --cov=acto
+      - name: Upload coverage
+        uses: actions/upload-artifact@v4
+        with:
+          name: .coverage.${{ github.sha }}.integration-test
+          path: .coverage.${{ github.sha }}.integration-test
+          retention-days: 1
+  coverage-report:
+    runs-on: ubuntu-latest
+    needs: [unittest, integration-test]
+    if: ${{ github.event_name == 'pull_request' }}
+    permissions:
+      pull-requests: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download unittest coverage
+        uses: actions/download-artifact@v4
+        with:
+          name: .coverage.${{ github.sha }}.unittest
+      - name: Download integration test coverage
+        uses: actions/download-artifact@v4
+        with:
+          name: .coverage.${{ github.sha }}.integration-test
+      - name: Merge coverage files
         run: |
-          pytest -m "not local and not singleBugReproduction" --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=acto | tee pytest-coverage.txt
-      - name: Pytest coverage comment
-        if: ${{ github.event == 'pull_request' }}
-        uses: MishaKav/pytest-coverage-comment@main
+          pip install coverage
+          coverage combine
+      - name: list files
+        run: ls -al
+      - name: Coverage comment
+        uses: py-cov-action/python-coverage-comment-action@v3
         with:
-          pytest-coverage-path: ./pytest-coverage.txt
-          junitxml-path: ./pytest.xml
+          GITHUB_TOKEN: ${{ github.token }}
+          ANNOTATE_MISSING_LINES: true

pyproject.toml

@@ -0,0 +1,11 @@
+[tool.autopep8]
+max_line_length = 80
+aggressive = 3
+experimental = true
+
+[tool.isort]
+profile = "black"
+line_length = 80
+
+[tool.coverage.run]
+relative_files = true

test/test_bug_reproduction.py → test/e2e_tests/test_bug_reproduction.py

@@ -3,21 +3,26 @@
 import pathlib
 import queue
 import random
-from typing import Dict, List, Tuple
 import unittest
+from test.utils import BugConfig, all_bugs, check_postdiff_runtime_error
+from typing import Dict, List, Tuple
 
 import pytest
+
 from acto.common import PassResult
 from acto.reproduce import reproduce, reproduce_postdiff
 
-from test.utils import BugConfig, all_bugs, check_postdiff_runtime_error
-
 test_dir = pathlib.Path(__file__).parent.resolve()
 test_data_dir = os.path.join(test_dir, 'test_data')
 
-def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_namespace: int) -> bool:
+
+def run_bug_config(
+        operator_name: str,
+        bug_id: str,
+        bug_config: BugConfig,
+        acto_namespace: int) -> bool:
     '''This function tries to reproduce a bug according to the bug config
-    
+
     Returns:
         if the reproduction is successful
     '''
@@ -32,18 +37,18 @@ def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_
                                   cluster_runtime='KIND',
                                   acto_namespace=acto_namespace)
     if bug_config.difftest:
-        if bug_config.diffdir != None:
+        if bug_config.diffdir is not None:
             diff_repro_dir = os.path.join(test_data_dir, bug_config.diffdir)
             work_dir = f'testrun-{bug_id}-diff'
             reproduce(work_dir,
-                        diff_repro_dir,
-                        operator_config,
-                        cluster_runtime='KIND',
-                        acto_namespace=acto_namespace)
+                      diff_repro_dir,
+                      operator_config,
+                      cluster_runtime='KIND',
+                      acto_namespace=acto_namespace)
         if reproduce_postdiff(work_dir,
-                                operator_config,
-                                cluster_runtime='KIND',
-                                acto_namespace=acto_namespace):
+                              operator_config,
+                              cluster_runtime='KIND',
+                              acto_namespace=acto_namespace):
             reproduced = True
         else:
             print(f"Bug {bug_id} not reproduced!")
@@ -52,10 +57,10 @@ def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_
     if bug_config.declaration:
         if len(normal_run_result) != 0:
             last_error = normal_run_result[-1]
-            if last_error.state_result != None and not isinstance(
+            if last_error.state_result is not None and not isinstance(
                     last_error.state_result, PassResult):
                 reproduced = True
-            elif last_error.recovery_result != None and not isinstance(
+            elif last_error.recovery_result is not None and not isinstance(
                     last_error.recovery_result, PassResult):
                 reproduced = True
         else:
@@ -65,10 +70,10 @@ def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_
     if bug_config.recovery:
         if len(normal_run_result) != 0:
             last_error = normal_run_result[-1]
-            if last_error.recovery_result != None and not isinstance(
+            if last_error.recovery_result is not None and not isinstance(
                     last_error.recovery_result, PassResult):
                 reproduced = True
-            elif last_error.state_result != None and not isinstance(
+            elif last_error.state_result is not None and not isinstance(
                     last_error.state_result, PassResult):
                 reproduced = True
         else:
@@ -80,7 +85,7 @@ def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_
             reproduced = True
         elif len(normal_run_result) != 0:
             last_error = normal_run_result[-1]
-            if last_error.health_result != None and not isinstance(
+            if last_error.health_result is not None and not isinstance(
                     last_error.health_result, PassResult):
                 reproduced = True
         else:
@@ -92,13 +97,17 @@ def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_
     else:
         return False
 
-def run_worker(workqueue: multiprocessing.Queue, acto_namespace: int, reproduction_results: Dict[str, bool]):
+
+def run_worker(
+        workqueue: multiprocessing.Queue, acto_namespace: int,
+        reproduction_results: Dict[str, bool]):
     while True:
         try:
-            bug_tuple: Tuple[str, str, BugConfig] = workqueue.get(block=True, timeout=5)
+            bug_tuple: Tuple[str, str, BugConfig] = workqueue.get(
+                block=True, timeout=5)
         except queue.Empty:
             break
-        
+
         operator_name, bug_id, bug_config = bug_tuple
         reproduced = False
         NUM_RETRY = 3
@@ -113,14 +122,15 @@ def run_worker(workqueue: multiprocessing.Queue, acto_namespace: int, reproducti
 
         reproduction_results[bug_id] = reproduced
 
-@pytest.mark.local
+
+@pytest.mark.all_bug_reproduction
 class TestBugReproduction(unittest.TestCase):
 
     def __init__(self, methodName: str = "runTest") -> None:
         super().__init__(methodName)
 
         # TODO: make _num_workers a command line argument
-        self._num_workers = 2 # Number of workers to run the test
+        self._num_workers = 2  # Number of workers to run the test
 
     def test_all_bugs(self):
         manager = multiprocessing.Manager()
@@ -130,11 +140,14 @@ def test_all_bugs(self):
             for bug_id, bug_config in bugs.items():
                 workqueue.put((operator, bug_id, bug_config))
 
-        reproduction_results: Dict[str, bool] = manager.dict() # workers write reproduction results
-                                                               # to this dict. Bug ID -> if success
+        # workers write reproduction results
+        reproduction_results: Dict[str, bool] = manager.dict()
+        # to this dict. Bug ID -> if success
         processes: List[multiprocessing.Process] = []
         for i in range(self._num_workers):
-            p = multiprocessing.Process(target=run_worker, args=(workqueue, i, reproduction_results))
+            p = multiprocessing.Process(
+                target=run_worker, args=(
+                    workqueue, i, reproduction_results))
             p.start()
             processes.append(p)
 
@@ -145,16 +158,17 @@ def test_all_bugs(self):
         for bug_id, if_reproduced in reproduction_results.items():
             if not if_reproduced:
                 errors.append(bug_id)
-        
+
         self.assertFalse(errors, f'Test failed with {errors}')
 
+
 @pytest.mark.singleBugReproduction
 class TestSingleBugReproduction(unittest.TestCase):
 
     def __init__(self, methodName: str = "runTest") -> None:
         super().__init__(methodName)
 
-        self._num_workers = 1 # Number of workers to run the test
+        self._num_workers = 1  # Number of workers to run the test
 
     def test_all_bugs(self):
         manager = multiprocessing.Manager()
@@ -164,11 +178,14 @@ def test_all_bugs(self):
         bug_id, bug_config = random.choice(list(bugs.items()))
         workqueue.put((operator, bug_id, bug_config))
 
-        reproduction_results: Dict[str, bool] = manager.dict() # workers write reproduction results
-                                                               # to this dict. Bug ID -> if success
+        # workers write reproduction results
+        reproduction_results: Dict[str, bool] = manager.dict()
+        # to this dict. Bug ID -> if success
         processes: List[multiprocessing.Process] = []
         for i in range(self._num_workers):
-            p = multiprocessing.Process(target=run_worker, args=(workqueue, i, reproduction_results))
+            p = multiprocessing.Process(
+                target=run_worker, args=(
+                    workqueue, i, reproduction_results))
             p.start()
             processes.append(p)
 
@@ -179,9 +196,9 @@ def test_all_bugs(self):
         for bug_id, if_reproduced in reproduction_results.items():
             if not if_reproduced:
                 errors.append(bug_id)
-        
+
         self.assertFalse(errors, f'Test failed with {errors}')
 
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()