[release] Accommodate multiple filters with same attribute (#57577)

and allow a test to match with one of filters in the same attribute
instead of saying it's mismatch if any of the filter fail

Signed-off-by: kevin <kevin@anyscale.com>

Author: khluu

release/ray_release/buildkite/filter.py

@@ -22,7 +22,7 @@ def _unflattened_lookup(lookup: Dict, flat_key: str, delimiter: str = "/") -> An
 def filter_tests(
     test_collection: List[Test],
     frequency: Frequency,
-    test_filters: Optional[Dict[str, str]] = None,
+    test_filters: Optional[Dict[str, list]] = None,
     prefer_smoke_tests: bool = False,
     run_jailed_tests: bool = False,
     run_unstable_tests: bool = False,
@@ -38,19 +38,28 @@ def filter_tests(
         if test.is_kuberay() and get_global_config()["kuberay_disabled"]:
             continue
 
-        # Check if any test attributes match filters
+        # Check if test attributes match filters
+        # Logic: OR within same attribute, AND across different attributes
         if test_filters:
-            for attr, value in test_filters.items():
-                # Only prefix filter doesn't use regex
-                if attr == "prefix":
-                    if not test.get_name().startswith(value):
-                        attr_mismatch = True
-                        break
-                else:  # Match filters using regex
-                    attr_value = _unflattened_lookup(test, attr) or ""
-                    if not re.match(value, attr_value):
-                        attr_mismatch = True
-                        break
+            for attr, values in test_filters.items():
+                # Check if at least one value matches for this attribute (OR logic)
+                attr_matched = False
+                for value in values:
+                    # Only prefix filter doesn't use regex
+                    if attr == "prefix":
+                        if test.get_name().startswith(value):
+                            attr_matched = True
+                            break
+                    else:  # Match filters using regex
+                        attr_value = _unflattened_lookup(test, attr) or ""
+                        if re.match(value, attr_value):
+                            attr_matched = True
+                            break
+
+                # If none of the values matched for this attribute, skip this test
+                if not attr_matched:
+                    attr_mismatch = True
+                    break
         if attr_mismatch:
             continue
 

release/ray_release/buildkite/settings.py

@@ -63,7 +63,7 @@ def get_priority(priority_str: str) -> Priority:
     return priority_str_to_enum[priority_str]
 
 
-def get_test_filters(filters_str: str) -> Dict[str, str]:
+def get_test_filters(filters_str: str) -> Dict[str, list]:
     if not filters_str:
         return {}
 
@@ -77,7 +77,10 @@ def get_test_filters(filters_str: str) -> Dict[str, str]:
             raise ReleaseTestConfigError(
                 f"Invalid test filter: {line}. " "Should be of the form attr:value"
             )
-        test_filters[parts[0]] = parts[1]
+        # Support multiple values for the same attribute (OR logic)
+        if parts[0] not in test_filters:
+            test_filters[parts[0]] = []
+        test_filters[parts[0]].append(parts[1])
     return test_filters
 
 

release/ray_release/tests/test_buildkite.py

@@ -114,16 +114,20 @@ def testGetTestAttrRegexFilters(self):
         self.assertDictEqual(test_filters, {})
 
         test_filters = get_test_filters("name:xxx")
-        self.assertDictEqual(test_filters, {"name": "xxx"})
+        self.assertDictEqual(test_filters, {"name": ["xxx"]})
 
         test_filters = get_test_filters("name:xxx\n")
-        self.assertDictEqual(test_filters, {"name": "xxx"})
+        self.assertDictEqual(test_filters, {"name": ["xxx"]})
 
         test_filters = get_test_filters("name:xxx\n\nteam:yyy")
-        self.assertDictEqual(test_filters, {"name": "xxx", "team": "yyy"})
+        self.assertDictEqual(test_filters, {"name": ["xxx"], "team": ["yyy"]})
 
         test_filters = get_test_filters("name:xxx\n \nteam:yyy\n")
-        self.assertDictEqual(test_filters, {"name": "xxx", "team": "yyy"})
+        self.assertDictEqual(test_filters, {"name": ["xxx"], "team": ["yyy"]})
+
+        # Test multiple filters with same attribute (OR logic)
+        test_filters = get_test_filters("name:xxx\nname:yyy")
+        self.assertDictEqual(test_filters, {"name": ["xxx", "yyy"]})
 
         with self.assertRaises(ReleaseTestConfigError):
             get_test_filters("xxx")
@@ -172,8 +176,8 @@ def testSettingsOverrideEnv(self):
         self.assertDictEqual(
             updated_settings["test_filters"],
             {
-                "name": "xxx",
-                "team": "yyy",
+                "name": ["xxx"],
+                "team": ["yyy"],
             },
         )
 
@@ -192,7 +196,7 @@ def testSettingsOverrideEnv(self):
             {
                 "frequency": Frequency.NIGHTLY,
                 "prefer_smoke_tests": False,
-                "test_filters": {"name": "name_filter"},
+                "test_filters": {"name": ["name_filter"]},
                 "ray_test_repo": "https://github.com/user/ray.git",
                 "ray_test_branch": "sub/branch",
                 "priority": Priority.MANUAL,
@@ -207,7 +211,7 @@ def testSettingsOverrideEnv(self):
             {
                 "frequency": Frequency.ANY,
                 "prefer_smoke_tests": True,
-                "test_filters": {"name": "name_filter"},
+                "test_filters": {"name": ["name_filter"]},
                 "ray_test_repo": "https://github.com/user/ray.git",
                 "ray_test_branch": "sub/branch",
                 "priority": Priority.MANUAL,
@@ -335,8 +339,8 @@ def testSettingsOverrideBuildkite(self):
             self.assertDictEqual(
                 updated_settings["test_filters"],
                 {
-                    "name": "xxx",
-                    "group": "yyy",
+                    "name": ["xxx"],
+                    "group": ["yyy"],
                 },
             )
 
@@ -354,7 +358,7 @@ def testSettingsOverrideBuildkite(self):
                 {
                     "frequency": Frequency.NIGHTLY,
                     "prefer_smoke_tests": False,
-                    "test_filters": {"name": "name_filter"},
+                    "test_filters": {"name": ["name_filter"]},
                     "ray_test_repo": "https://github.com/user/ray.git",
                     "ray_test_branch": "sub/branch",
                     "priority": Priority.MANUAL,
@@ -370,7 +374,7 @@ def testSettingsOverrideBuildkite(self):
                 {
                     "frequency": Frequency.ANY,
                     "prefer_smoke_tests": True,
-                    "test_filters": {"name": "name_filter"},
+                    "test_filters": {"name": ["name_filter"]},
                     "ray_test_repo": "https://github.com/user/ray.git",
                     "ray_test_branch": "sub/branch",
                     "priority": Priority.MANUAL,
@@ -432,7 +436,7 @@ def testFilterTests(self, *args):
 
         # Test filter by prefix alone
         filtered = self._filter_names(
-            tests, frequency=Frequency.ANY, test_filters={"prefix": "test"}
+            tests, frequency=Frequency.ANY, test_filters={"prefix": ["test"]}
         )
         self.assertSequenceEqual(
             filtered,
@@ -448,7 +452,7 @@ def testFilterTests(self, *args):
         filtered = self._filter_names(
             tests,
             frequency=Frequency.NIGHTLY,
-            test_filters={"prefix": "test", "name": "other.*"},
+            test_filters={"prefix": ["test"], "name": ["other.*"]},
         )
         self.assertSequenceEqual(
             filtered,
@@ -522,7 +526,7 @@ def testFilterTests(self, *args):
         filtered = self._filter_names(
             tests,
             frequency=Frequency.NIGHTLY,
-            test_filters={"name": "other.*"},
+            test_filters={"name": ["other.*"]},
         )
         self.assertSequenceEqual(
             filtered,
@@ -534,7 +538,7 @@ def testFilterTests(self, *args):
         filtered = self._filter_names(
             tests,
             frequency=Frequency.NIGHTLY,
-            test_filters={"name": "test.*"},
+            test_filters={"name": ["test.*"]},
         )
         self.assertSequenceEqual(
             filtered,
@@ -547,7 +551,7 @@ def testFilterTests(self, *args):
         )
 
         filtered = self._filter_names(
-            tests, frequency=Frequency.NIGHTLY, test_filters={"name": "test"}
+            tests, frequency=Frequency.NIGHTLY, test_filters={"name": ["test"]}
         )
         self.assertSequenceEqual(
             filtered,
@@ -562,22 +566,41 @@ def testFilterTests(self, *args):
         filtered = self._filter_names(
             tests,
             frequency=Frequency.NIGHTLY,
-            test_filters={"name": "test.*", "team": "team_1"},
+            test_filters={"name": ["test.*"], "team": ["team_1"]},
         )
         self.assertSequenceEqual(filtered, [("test_1", False)])
 
         filtered = self._filter_names(
             tests,
             frequency=Frequency.NIGHTLY,
-            test_filters={"name": "test_1|test_2"},
+            test_filters={"name": ["test_1|test_2"]},
         )
         self.assertSequenceEqual(filtered, [("test_1", False), ("test_2", True)])
 
+        # Test OR logic within same attribute
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            test_filters={"name": ["^test_1$", "^test_3$"]},
+        )
+        self.assertSequenceEqual(filtered, [("test_1", False), ("test_3", False)])
+
+        # Test OR logic with AND across attributes
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            test_filters={
+                "name": ["^test_1$", "^test_3$"],
+                "team": ["team_1", "team_2"],
+            },
+        )
+        self.assertSequenceEqual(filtered, [("test_1", False), ("test_3", False)])
+
         # Filter by nested properties
         filtered = self._filter_names(
             tests,
             frequency=Frequency.ANY,
-            test_filters={"run/type": "job"},
+            test_filters={"run/type": ["job"]},
         )
         self.assertSequenceEqual(
             filtered, [("test_1", False), ("other_2", False), ("test_4.kuberay", False)]
@@ -586,14 +609,14 @@ def testFilterTests(self, *args):
         filtered = self._filter_names(
             tests,
             frequency=Frequency.ANY,
-            test_filters={"run/type": "client"},
+            test_filters={"run/type": ["client"]},
         )
         self.assertSequenceEqual(filtered, [("test_2", False)])
 
         filtered = self._filter_names(
             tests,
             frequency=Frequency.ANY,
-            test_filters={"run/invalid": "xxx"},
+            test_filters={"run/invalid": ["xxx"]},
         )
         self.assertSequenceEqual(filtered, [])