Prefer automated tests for css tests with interact flag

For tests with "interact" flag that are also marked as reftests or testharness.js tests, prefer running the test automatically vs assuming it's a manual test. It has been verified that this is reasonable based on an insepction of the existing tests.
w3c · Apr 4, 2017 · d673bdc · d673bdc
1 parent 8b4ee50
commit d673bdc
Show file tree

Hide file tree

Showing 2 changed files with 94 additions and 4 deletions.
diff --git a/manifest/sourcefile.py b/manifest/sourcefile.py
@@ -429,13 +429,23 @@ def css_flags(self):
         return rv
 
     @cached_property
-    def content_is_css_manual(self):
+    def content_is_css_manual_strong(self):
         """Boolean indicating whether the file content represents a
-        CSS WG-style manual test"""
+        CSS WG-style manual test, in a way that implies it's manual even
+        if it otherwise looks like a reftest or testharness test"""
         if self.root is None:
             return None
         # return True if the intersection between the two sets is non-empty
-        return bool(self.css_flags & {"animated", "font", "history", "interact", "paged", "speech", "userstyle"})
+        return bool(self.css_flags & {"animated", "font", "history", "paged", "speech", "userstyle"})
+
+    @cached_property
+    def content_is_css_manual_weak(self):
+        """Boolean indicating whether the file content represents a
+        CSS WG-style manual test"""
+        if self.root is None:
+            return None
+        # Interaective tests that are actually other tests are not manual
+        return bool(self.css_flags & {"interact"})
 
     @cached_property
     def spec_link_nodes(self):
@@ -516,7 +526,7 @@ def manifest_items(self):
             rv = WebdriverSpecTest.item_type, [WebdriverSpecTest(self, self.url,
                                                                  timeout=self.timeout)]
 
-        elif self.content_is_css_manual and not self.name_is_reference:
+        elif self.content_is_css_manual_strong and not self.name_is_reference:
             rv = ManualTest.item_type, [ManualTest(self, self.url)]
 
         elif self.content_is_testharness:
@@ -530,6 +540,9 @@ def manifest_items(self):
                   [RefTestNode(self, self.url, self.references, timeout=self.timeout,
                                viewport_size=self.viewport_size, dpi=self.dpi)])
 
+        elif self.content_is_css_manual_weak and not self.name_is_reference:
+            rv = ManualTest.item_type, [ManualTest(self, self.url)]
+
         elif self.content_is_css_visual and not self.name_is_reference:
             rv = VisualTest.item_type, [VisualTest(self, self.url)]
 

diff --git a/manifest/tests/test_sourcefile.py b/manifest/tests/test_sourcefile.py
@@ -472,6 +472,83 @@ def test_reftest_node(ext):
     assert items(s) == [("reftest_node", "/" + filename)]
 
 
+@pytest.mark.parametrize("flags", ["animated", "font", "history", "paged", "speech", "userstyle",
+                                   ("interact", "paged")])
+def test_css_manual_strong(flags):
+    if isinstance(flags, tuple):
+        flags = " ".join(flags)
+
+    content = b"""
+<html>
+<head>
+<link rel="help" href="http://www.w3.org/TR/CSS21/box.html#bidi-box-model">
+<link rel="match" href="test-ref.htm">
+<meta name="flags" content="%s">
+</head>
+<body></body>
+</html>
+""" % flags
+
+    filename = "css/test.htm"
+    s = create(filename, content)
+
+    assert s.content_is_ref_node
+    assert s.content_is_css_manual_strong
+
+    assert items(s) == [("manual", "/" + filename)]
+
+
+@pytest.mark.parametrize("flags", ["interact", ("interact", "ahem")])
+def test_not_css_manual_strong(flags):
+    if isinstance(flags, tuple):
+        flags = " ".join(flags)
+
+    content = b"""
+<html>
+<head>
+<link rel="help" href="http://www.w3.org/TR/CSS21/box.html#bidi-box-model">
+<link rel="match" href="test-ref.htm">
+<meta name="flags" content="%s">
+</head>
+<body></body>
+</html>
+""" % flags
+
+    filename = "css/test.htm"
+    s = create(filename, content)
+
+    assert s.content_is_ref_node
+    assert not s.content_is_css_manual_strong
+    assert s.content_is_css_manual_weak
+
+    assert items(s) == [("reftest_node", "/" + filename)]
+
+
+@pytest.mark.parametrize("flags", ["interact", ("interact", "ahem")])
+def test_css_manual_weak(flags):
+    if isinstance(flags, tuple):
+        flags = " ".join(flags)
+
+    content = b"""
+<html>
+<head>
+<link rel="help" href="http://www.w3.org/TR/CSS21/box.html#bidi-box-model">
+<meta name="flags" content="%s">
+</head>
+<body></body>
+</html>
+""" % flags
+
+    filename = "css/test.htm"
+    s = create(filename, content)
+
+    assert not s.content_is_ref_node
+    assert not s.content_is_css_manual_strong
+    assert s.content_is_css_manual_weak
+
+    assert items(s) == [("manual", "/" + filename)]
+
+
 @pytest.mark.parametrize("ext", ["xht", "html", "xhtml", "htm", "xml", "svg"])
 def test_css_visual(ext):
     content = b"""