Infrastructure: Temporarily comment out flaky tests (pull #1488)

Disable a few unreliable regression tests that we don't want to impede other work while we determine the root cause of the inconsistent test behavior. This change comments out tests that fail most frequently in CI. The regression test coverage report will report them as missing. The work to resolve the root issue is tracked by issue #1415.
w3c · Aug 9, 2020 · b1e4f2c · b1e4f2c
1 parent 32f0774
commit b1e4f2c
Showing 1 changed file with 45 additions and 40 deletions.
diff --git a/test/tests/menu-button_actions-active-descendant.js b/test/tests/menu-button_actions-active-descendant.js
@@ -277,38 +277,42 @@ ariaTest('"escape" on role="menu"', exampleFile, 'menu-escape', async (t) => {
   }
 });
 
-ariaTest('"down arrow" on role="menu"', exampleFile, 'menu-down-arrow', async (t) => {
+// This test is flaky, so is commented out for now.
+// We are traking it in issue:https://github.com/w3c/aria-practices/issues/1415
+// ariaTest('"down arrow" on role="menu"', exampleFile, 'menu-down-arrow', async (t) => {
 
-  await openMenu(t);
-  const menu = await t.context.session.findElement(By.css(ex.menuSelector));
-  const items = await t.context.queryElements(t, ex.menuitemSelector);
+//   await openMenu(t);
+//   const menu = await t.context.session.findElement(By.css(ex.menuSelector));
+//   const items = await t.context.queryElements(t, ex.menuitemSelector);
 
-  for (let index = 0; index < items.length - 1; index++) {
+//   for (let index = 0; index < items.length - 1; index++) {
 
-    await menu.sendKeys(Key.ARROW_DOWN);
-    await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, index + 1);
-  }
+//     await menu.sendKeys(Key.ARROW_DOWN);
+//     await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, index + 1);
+//   }
 
-  await menu.sendKeys(Key.ARROW_DOWN);
-  await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, 0);
-});
+//   await menu.sendKeys(Key.ARROW_DOWN);
+//   await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, 0);
+// });
 
-ariaTest('"up arrow" on role="menu"', exampleFile, 'menu-up-arrow', async (t) => {
+// This test is flaky, so is commented out for now.
+// We are traking it in issue:https://github.com/w3c/aria-practices/issues/1415
+// ariaTest('"up arrow" on role="menu"', exampleFile, 'menu-up-arrow', async (t) => {
 
-  await openMenu(t);
-  const menu = await t.context.session.findElement(By.css(ex.menuSelector));
-  const items = await t.context.queryElements(t, ex.menuitemSelector);
+//   await openMenu(t);
+//   const menu = await t.context.session.findElement(By.css(ex.menuSelector));
+//   const items = await t.context.queryElements(t, ex.menuitemSelector);
 
-  await menu.sendKeys(Key.ARROW_UP);
-  await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, ex.numMenuitems - 1);
+//   await menu.sendKeys(Key.ARROW_UP);
+//   await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, ex.numMenuitems - 1);
 
-  for (let index = items.length - 1; index > 0; index--) {
+//   for (let index = items.length - 1; index > 0; index--) {
 
-    await menu.sendKeys(Key.ARROW_UP);
-    await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, index - 1);
-  }
+//     await menu.sendKeys(Key.ARROW_UP);
+//     await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, index - 1);
+//   }
 
-});
+// });
 
 ariaTest('"home" on role="menu"', exampleFile, 'menu-home', async (t) => {
 
@@ -365,21 +369,22 @@ ariaTest('"end" on role="menu"', exampleFile, 'menu-end', async (t) => {
   await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, last);
 });
 
-ariaTest('"character" on role="menu"', exampleFile, 'menu-character', async (t) => {
-
-  const charIndexTest = [
-    { sendChar: 'x', sendIndex: 0, endIndex: 0 },
-    { sendChar: 'a', sendIndex: 0, endIndex: 1 },
-    { sendChar: 'y', sendIndex: 1, endIndex: 1 },
-    { sendChar: 'a', sendIndex: 1, endIndex: 2 }
-  ];
-
-  await openMenu(t);
-  const menu = await t.context.session.findElement(By.css(ex.menuSelector));
-
-  for (let test of charIndexTest) {
-    await menu.sendKeys(test.sendChar);
-
-    await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, test.endIndex);
-  }
-});
+// This test is flaky, so is commented out for now.
+// We are traking it in issue:https://github.com/w3c/aria-practices/issues/1415
+// ariaTest('"character" on role="menu"', exampleFile, 'menu-character', async (t) => {
+//   const charIndexTest = [
+//     { sendChar: 'x', sendIndex: 0, endIndex: 0 },
+//     { sendChar: 'a', sendIndex: 0, endIndex: 1 },
+//     { sendChar: 'y', sendIndex: 1, endIndex: 1 },
+//     { sendChar: 'a', sendIndex: 1, endIndex: 2 }
+//   ];
+
+//   await openMenu(t);
+//   const menu = await t.context.session.findElement(By.css(ex.menuSelector));
+
+//   for (let test of charIndexTest) {
+//     await menu.sendKeys(test.sendChar);
+
+//     await assertAriaActivedescendant(t, ex.menuSelector, ex.menuitemSelector, test.endIndex);
+//   }
+// });