Skip all flaky tests with GitHub issues.

This brings our skip count up from 7 python and 2 rust to 25 python and
4 rust, hopefully in the name of stability as outlined in pantsbuild#9386.

Also updates the How to Contribute docs with a pointer to pantsbuild#9386 for
the latest procedure.

Includes flaky tests from:
 pantsbuild#9313 pantsbuild#9312 pantsbuild#8679 pantsbuild#8678 pantsbuild#8520 pantsbuild#8520 pantsbuild#8520 pantsbuild#8405 pantsbuild#8193 pantsbuild#8171 pantsbuild#7836 pantsbuild#7622
 pantsbuild#7573 pantsbuild#7572 pantsbuild#7554 pantsbuild#7547 pantsbuild#7457 pantsbuild#7168 pantsbuild#7149 pantsbuild#6838 pantsbuild#6829 pantsbuild#6787 pantsbuild#6114

[ci skip-jvm-tests]  # No JVM changes made.

src/docs/howto_develop.md

@@ -189,10 +189,9 @@ not your fork. If you are posting a review request, put the pull request number
 field. Then, when you close the request, you can navigate from the bug number to easily close
 the pull request.
 
-If your CI-build failed in Travis-CI, and the failure looks like it's not due to
-your change, please open an issue with the part of the CI log containing the test failure and label
-the issue with `flaky-test`. If an issue already exists, add a comment to it noting that you
-encountered it too. After you've done that, you can ask in slack for someone to restart the shard.
+If your CI-build failed in Travis-CI, and the failure looks like it's not due to your change, please
+go to https://github.com/pantsbuild/pants/issues/9386 and follow the instructions there to deal with
+the misbehaving test. After you've done that, you can ask in slack for someone to restart the shard.
 That will cause the shard to re-run its tests.
 
 #### Advanced: Remote execution of tests

src/rust/engine/process_execution/src/remote_tests.rs

@@ -1145,7 +1145,7 @@ async fn timeout_after_sufficiently_delayed_getoperations() {
   assert_cancellation_requests(&mock_server, vec![op_name.to_owned()]);
 }
 
-#[ignore] // https://github.com/pantsbuild/pants/issues/8405
+#[ignore] // flaky: https://github.com/pantsbuild/pants/issues/8405
 #[tokio::test]
 async fn dropped_request_cancels() {
   let request_timeout = Duration::new(10, 0);
@@ -1229,6 +1229,7 @@ async fn dropped_request_cancels() {
   assert_cancellation_requests(&mock_server, vec![op_name.to_owned()]);
 }
 
+#[ignore] // flaky: https://github.com/pantsbuild/pants/issues/7149
 #[tokio::test]
 async fn retry_for_cancelled_channel() {
   let execute_request = echo_foo_request();

src/rust/engine/serverset/src/tests.rs

@@ -100,6 +100,7 @@ async fn reattempts_unhealthy() {
   expect_both(&s, 3).await
 }
 
+#[ignore] // flaky: https://github.com/pantsbuild/pants/issues/7836
 #[tokio::test]
 async fn backoff_when_unhealthy() {
   let s = Serverset::new(

tests/python/pants_test/backend/graph_info/tasks/test_list_targets.py

@@ -112,7 +112,7 @@ def test_list_descendants(self):
             "a/b:b", "a/b/c:c", "a/b/c:c2", "a/b/c:c3", "a/b/d:d", "a/b/e:e1", args=["a/b::"]
         )
 
-    @pytest.mark.flaky(retries=1)  # https://github.com/pantsbuild/pants/issues/8678
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/8678")
     def test_list_all(self):
         self.assert_entries(
             "\n",

tests/python/pants_test/backend/jvm/subsystems/test_jar_dependency_management_integration.py

@@ -5,6 +5,8 @@
 from collections import namedtuple
 from contextlib import contextmanager
 
+import pytest
+
 from pants.testutil.pants_run_integration_test import PantsRunIntegrationTest
 from pants.util.contextutil import temporary_dir
 
@@ -198,6 +200,7 @@ def test_unit_tests_with_different_sets(self):
         self.assert_failure(run)
         self.assertIn("Undefined revs for jars unsupported by Coursier", run.stdout_data)
 
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/9313")
     def test_unit_tests_with_different_sets_one_batch(self):
         run = self.run_pants(
             [

tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_cache_compile_integration.py

@@ -7,6 +7,8 @@
 from textwrap import dedent
 from typing import Callable, Dict, List
 
+import pytest
+
 from pants.backend.jvm.tasks.jvm_compile.rsc.rsc_compile import RscCompile
 from pants.base.build_environment import get_buildroot
 from pants.fs.archive import TGZ, ZIP
@@ -290,6 +292,7 @@ def config(incremental_caching):
             Compile({srcfile: "public final class A {}"}, config(True), 2),
         )
 
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/9312")
     def test_rsc_and_zinc_caching(self):
         """Tests that with rsc-and-zinc, we write both artifacts."""
 

tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_java_compile_integration.py

@@ -226,7 +226,7 @@ def test_java_compile_with_different_resolved_jars_produce_different_artifacts(s
             self.assertEqual(guava_16_artifact_dir, guava_15_artifact_dir)
             self.assertEqual(len(os.listdir(guava_16_artifact_dir)), 2)
 
-    @pytest.mark.flaky(retries=1)  # https://github.com/pantsbuild/pants/issues/8171
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/8171")
     def test_java_compile_with_corrupt_remote(self):
         """Tests that a corrupt artifact in the remote cache still results in a successful
         compile."""

tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py

@@ -1,6 +1,8 @@
 # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import pytest
+
 from pants_test.backend.jvm.tasks.jvm_compile.rsc.rsc_compile_integration_base import (
     RscCompileIntegrationBase,
     ensure_compile_rsc_execution_strategy,
@@ -24,6 +26,7 @@ def test_executing_multi_target_binary(self):
         pants_run = self.do_command("run", "examples/src/scala/org/pantsbuild/example/hello/exe")
         self.assertIn("Hello, Resource World!", pants_run.stdout_data)
 
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/8679")
     @ensure_compile_rsc_execution_strategy(RscCompileIntegrationBase.rsc_and_zinc)
     def test_java_with_transitive_exported_scala_dep(self):
         self.do_command(

tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration_youtline.py

@@ -14,7 +14,7 @@ class RscCompileIntegrationYoutline(RscCompileIntegrationBase):
     def test_basic_binary(self):
         self._testproject_compile("mutual", "bin", "A")
 
-    @pytest.mark.flaky(retries=1)
+    @pytest.mark.skip(reason="flaky")
     @ensure_compile_rsc_execution_strategy(RscCompileIntegrationBase.outline_and_zinc)
     def test_public_inference(self):
         self._testproject_compile("public_inference", "public_inference", "PublicInference")

tests/python/pants_test/backend/jvm/test_backend_independence_integration.py

@@ -1,6 +1,8 @@
 # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import pytest
+
 from pants.testutil.pants_run_integration_test import PantsRunIntegrationTest
 
 
@@ -11,6 +13,7 @@ class BackendIndependenceTest(PantsRunIntegrationTest):
     def hermetic(cls):
         return True
 
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/7168")
     def test_independent_test_run(self):
         pants_run = self.run_pants(
             command=["test", "examples/tests/java/org/pantsbuild/example/hello/greet"],

tests/python/pants_test/cache/test_artifact_cache.py

@@ -99,15 +99,15 @@ def test_local_cache_with_seperate_extraction_root(self):
         with self.setup_local_cache(seperate_extraction_root=True) as artifact_cache:
             self.do_test_artifact_cache(artifact_cache)
 
-    @pytest.mark.flaky(retries=1)  # https://github.com/pantsbuild/pants/issues/6838
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6838")
     def test_restful_cache(self):
         with self.assertRaises(InvalidRESTfulCacheProtoError):
             RESTfulArtifactCache("foo", BestUrlSelector(["ftp://localhost/bar"]), "foo")
 
         with self.setup_rest_cache() as artifact_cache:
             self.do_test_artifact_cache(artifact_cache)
 
-    @pytest.mark.flaky(retries=1)  # https://github.com/pantsbuild/pants/issues/6838
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6838")
     def test_restful_cache_failover(self):
         bad_url = "http://badhost:123"
 
@@ -239,7 +239,7 @@ def test_local_backed_remote_cache_corrupt_artifact(self):
                         self.assertTrue(os.path.exists(results_dir))
                         self.assertTrue(len(os.listdir(results_dir)) == 0)
 
-    @pytest.mark.flaky(retries=1)  # https://github.com/pantsbuild/pants/issues/6838
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6838")
     def test_multiproc(self):
         key = CacheKey("muppet_key", "fake_hash")
 
@@ -265,7 +265,7 @@ def test_failed_multiproc(self):
                 call_insert((cache, key, [path], False))
                 self.assertFalse(call_use_cached_files((cache, key, None)))
 
-    @pytest.mark.flaky(retries=1)  # https://github.com/pantsbuild/pants/issues/6838
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6838")
     def test_noops_after_max_retries_exceeded(self):
         key = CacheKey("muppet_key", "fake_hash")
 

tests/python/pants_test/engine/legacy/test_graph_integration.py

@@ -114,6 +114,7 @@ def setup_bundle_target(self) -> Iterator[None]:
         with self.with_overwritten_file_content(build_path, f"{original_content}\n{new_content}"):
             yield
 
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/8520")
     def test_missing_sources_warnings(self):
         target_to_unmatched_globs = {
             "missing-globs": ["*.a"],
@@ -144,6 +145,7 @@ def test_missing_sources_warnings(self):
                         in pants_run.stderr_data
                     )
 
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/8520")
     def test_existing_sources(self):
         target_full = f"{self._SOURCES_TARGET_BASE}:text"
         pants_run = self.run_pants(
@@ -178,6 +180,7 @@ def test_missing_bundles_warnings(self):
                 in pants_run.stderr_data
             )
 
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/8520")
     def test_existing_bundles(self):
         target_full = f"{self._BUNDLE_TARGET_BASE}:mapper"
         pants_run = self.run_pants(
@@ -192,7 +195,7 @@ def test_existing_directory_with_no_build_files_fails(self):
         self.assert_failure(pants_run)
         self.assertIn("does not match any targets.", pants_run.stderr_data)
 
-    @unittest.skip("Flaky: https://github.com/pantsbuild/pants/issues/6787")
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/6787")
     def test_error_message(self):
         with self.setup_bundle_target(), self.setup_sources_targets():
             for target in self._ERR_TARGETS:

tests/python/pants_test/engine/test_engine.py

@@ -223,9 +223,7 @@ def fork_context_body():
         res = self.mk_scheduler().with_fork_context(fork_context_body)
         self.assertEquals(res, expected)
 
-    @unittest.skip(
-        "Inherently flaky as described in https://github.com/pantsbuild/pants/issues/6829"
-    )
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/6829")
     def test_trace_multi(self):
         # Tests that when multiple distinct failures occur, they are each rendered.
 

tests/python/pants_test/pantsd/test_pantsd_integration.py

@@ -50,7 +50,7 @@ def test_pantsd_compile(self):
             pantsd_run(["compile", "examples/src/scala/org/pantsbuild/example/hello/welcome"])
             checker.assert_started()
 
-    @unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7573")
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7573")
     def test_pantsd_run(self):
         with self.pantsd_successful_run_context("debug") as (
             pantsd_run,
@@ -124,7 +124,7 @@ def test_pantsd_pantsd_runner_doesnt_die_after_failed_run(self):
                 )
                 checker.assert_running()
 
-    @pytest.mark.flaky(retries=1)  # https://github.com/pantsbuild/pants/issues/6114
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6114")
     def test_pantsd_lifecycle_invalidation(self):
         """Runs pants commands with pantsd enabled, in a loop, alternating between options that
         should invalidate pantsd and incur a restart and then asserts for pid consistency."""
@@ -216,7 +216,7 @@ def test_pantsd_aligned_output(self):
         for run_pairs in zip(non_daemon_runs, daemon_runs):
             self.assertEqual(*(run.stdout_data for run in run_pairs))
 
-    @unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7622")
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7622")
     def test_pantsd_filesystem_invalidation(self):
         """Runs with pantsd enabled, in a loop, while another thread invalidates files."""
         with self.pantsd_successful_run_context() as (pantsd_run, checker, workdir, _):
@@ -385,7 +385,7 @@ def test_pantsd_pid_change(self):
             # Remove the pidfile so that the teardown script doesn't try to kill process 9.
             os.unlink(pidpath)
 
-    @pytest.mark.flaky(retries=1)  # https://github.com/pantsbuild/pants/issues/8193
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/8193")
     def test_pantsd_memory_usage(self):
         """Validates that after N runs, memory usage has increased by no more than X percent."""
         number_of_runs = 10
@@ -542,7 +542,7 @@ def _assert_pantsd_keyboardinterrupt_signal(self, signum, regexps=[], quit_timeo
                 # The pantsd processes should be dead, and they should have exited with 1.
                 self.assertFalse(proc.is_running())
 
-    @unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7554")
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7554")
     def test_pantsd_sigterm(self):
         self._assert_pantsd_keyboardinterrupt_signal(
             signal.SIGTERM,
@@ -560,7 +560,7 @@ def test_pantsd_sigterm(self):
             ],
         )
 
-    @unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7572")
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7572")
     def test_pantsd_sigquit(self):
         self._assert_pantsd_keyboardinterrupt_signal(
             signal.SIGQUIT,
@@ -578,7 +578,7 @@ def test_pantsd_sigquit(self):
             ],
         )
 
-    @unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7547")
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7547")
     def test_pantsd_sigint(self):
         self._assert_pantsd_keyboardinterrupt_signal(
             signal.SIGINT,
@@ -592,7 +592,7 @@ def test_pantsd_sigint(self):
             ],
         )
 
-    @unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7457")
+    @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7457")
     def test_signal_pailgun_stream_timeout(self):
         # NB: The actual timestamp has the date and time at sub-second granularity. The date is just
         # used here since that is known in advance in order to assert that the timestamp is well-formed.

tests/python/pants_test/pantsd/test_process_manager.py

@@ -144,7 +144,7 @@ def test_readwrite_metadata_by_name(self):
                 self.pmm.read_metadata_by_name(self.NAME, self.TEST_KEY, int), self.TEST_VALUE_INT
             )
 
-    @pytest.mark.flaky(retries=1)  # https://github.com/pantsbuild/pants/issues/6836
+    @pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6836")
     def test_deadline_until(self):
         with self.assertRaises(ProcessMetadataManager.Timeout):
             with self.captured_logging(logging.INFO) as captured: