[ci] add a initial version of CI test state machine (ray-project#42280)

A initial version of CI test state machine with some simple movement

Signed-off-by: can <can@anyscale.com>

release/ray_release/test_automation/ci_state_machine.py

@@ -0,0 +1,45 @@
+from ray_release.test_automation.state_machine import (
+    TestStateMachine,
+    DEFAULT_ISSUE_OWNER,
+)
+
+from ray_release.test import Test, TestState
+
+
+class CITestStateMachine(TestStateMachine):
+    def _move_hook(self, from_state: TestState, to_state: TestState) -> None:
+        change = (from_state, to_state)
+        if change == (TestState.PASSING, TestState.CONSITENTLY_FAILING):
+            self._create_github_issue()
+        elif change == (TestState.FAILING, TestState.CONSITENTLY_FAILING):
+            self._create_github_issue()
+        elif change == (TestState.CONSITENTLY_FAILING, TestState.PASSING):
+            self._close_github_issue()
+
+    def _state_hook(self, state: TestState) -> None:
+        pass
+
+    def _create_github_issue(self) -> None:
+        labels = [
+            "P0",
+            "bug",
+            "ci-test",
+            "ray-test-bot",
+            "triage",
+            self.test.get_oncall(),
+        ]
+        issue_number = self.ray_repo.create_issue(
+            title=f"CI test {self.test.get_name()} failed",
+            body=(
+                f"CI test **{self.test.get_name()}** failed. "
+                f"See {self.test_results[0].url} for more details.\n\n"
+                f"Managed by OSS Test Policy"
+            ),
+            labels=labels,
+            assignee=DEFAULT_ISSUE_OWNER,
+        ).number
+        self.test[Test.KEY_GITHUB_ISSUE_NUMBER] = issue_number
+
+    def _consistently_failing_to_jailed(self) -> bool:
+        # CI tests go to flaky first after failed instead of jailed
+        return False

release/ray_release/test_automation/release_state_machine.py

@@ -1,6 +1,9 @@
 from datetime import datetime, timedelta
 
-from ray_release.test_automation.state_machine import TestStateMachine
+from ray_release.test_automation.state_machine import (
+    TestStateMachine,
+    DEFAULT_ISSUE_OWNER,
+)
 from ray_release.test import Test, TestState
 from ray_release.logger import logger
 
@@ -67,7 +70,7 @@ def _create_github_issue(self) -> None:
                 f"Managed by OSS Test Policy"
             ),
             labels=labels,
-            assignee="can-anyscale",
+            assignee=DEFAULT_ISSUE_OWNER,
         ).number
         self.test[Test.KEY_GITHUB_ISSUE_NUMBER] = issue_number
 

release/ray_release/test_automation/state_machine.py

@@ -12,6 +12,7 @@
 RAY_REPO = "ray-project/ray"
 AWS_SECRET_GITHUB = "ray_ci_github_token"
 AWS_SECRET_BUILDKITE = "ray_ci_buildkite_token"
+DEFAULT_ISSUE_OWNER = "can-anyscale"
 
 
 class TestStateMachine(abc.ABC):

release/ray_release/tests/test_state_machine.py

@@ -13,6 +13,7 @@
     ResultStatus,
 )
 from ray_release.test_automation.release_state_machine import ReleaseTestStateMachine
+from ray_release.test_automation.ci_state_machine import CITestStateMachine
 from ray_release.test_automation.state_machine import TestStateMachine
 
 
@@ -88,7 +89,58 @@ def jobs(self):
 TestStateMachine.ray_buildkite = MockBuildkite()
 
 
-def test_move_from_passing_to_failing():
+def test_ci_move_from_passing_to_failing():
+    """
+    Test the entire lifecycle of a CI test when it moves from passing to failing.
+    Check that the conditions are met for each state transition. Also check that
+    gihub issues are created and closed correctly.
+    """
+    test = Test(name="test", team="ci")
+    # start from passing
+    assert test.get_state() == TestState.PASSING
+
+    # passing to failing
+    test.test_results = [
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+    ]
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.FAILING
+
+    # failing to consistently failing
+    test.test_results.extend(
+        [
+            TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+            TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+        ]
+    )
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.CONSITENTLY_FAILING
+    issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
+    assert issue.state == "open"
+    assert "ci-test" in [label.name for label in issue.labels]
+
+    # never move from consistently failing to jailed
+    test.test_results.extend(
+        [
+            TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+            TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+        ]
+    )
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.CONSITENTLY_FAILING
+
+    # go back to passing
+    test.test_results.insert(
+        0,
+        TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
+    )
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.PASSING
+    assert test.get(Test.KEY_GITHUB_ISSUE_NUMBER) is None
+    assert issue.state == "closed"
+
+
+def test_release_move_from_passing_to_failing():
     test = Test(name="test", team="ci")
     # Test original state
     test.test_results = [
@@ -117,7 +169,7 @@ def test_move_from_passing_to_failing():
     assert test[Test.KEY_GITHUB_ISSUE_NUMBER] == MockIssueDB.issue_id - 1
 
 
-def test_move_from_failing_to_consisently_failing():
+def test_release_move_from_failing_to_consisently_failing():
     test = Test(name="test", team="ci", stable=False)
     test[Test.KEY_BISECT_BUILD_NUMBER] = 1
     test.test_results = [
@@ -138,7 +190,7 @@ def test_move_from_failing_to_consisently_failing():
     assert "unstable-release-test" in labels
 
 
-def test_move_from_failing_to_passing():
+def test_release_move_from_failing_to_passing():
     test = Test(name="test", team="ci")
     test.test_results = [
         TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
@@ -160,7 +212,7 @@ def test_move_from_failing_to_passing():
     assert test.get(Test.KEY_BISECT_BLAMED_COMMIT) is None
 
 
-def test_move_from_failing_to_jailed():
+def test_release_move_from_failing_to_jailed():
     test = Test(name="test", team="ci")
     test.test_results = [
         TestResult.from_result(Result(status=ResultStatus.ERROR.value)),