Skip to content

Commit

Permalink
Skip all flaky tests with GitHub issues.
Browse files Browse the repository at this point in the history
This brings our skip count up from 7 python and 2 rust to 25 python and
4 rust, hopefully in the name of stability as outlined in pantsbuild#9386.

Also updates the How to Contribute docs with a pointer to pantsbuild#9386 for
the latest procedure.

Includes flaky tests from:
 pantsbuild#9313 pantsbuild#9312 pantsbuild#8679 pantsbuild#8678 pantsbuild#8520 pantsbuild#8520 pantsbuild#8520 pantsbuild#8405 pantsbuild#8193 pantsbuild#8171 pantsbuild#7836 pantsbuild#7622
 pantsbuild#7573 pantsbuild#7572 pantsbuild#7554 pantsbuild#7547 pantsbuild#7457 pantsbuild#7168 pantsbuild#7149 pantsbuild#6838 pantsbuild#6829 pantsbuild#6787 pantsbuild#6114

[ci skip-jvm-tests]  # No JVM changes made.
  • Loading branch information
jsirois committed Mar 25, 2020
1 parent 9cc097c commit ec40520
Show file tree
Hide file tree
Showing 15 changed files with 39 additions and 25 deletions.
7 changes: 3 additions & 4 deletions src/docs/howto_develop.md
Original file line number Diff line number Diff line change
Expand Up @@ -189,10 +189,9 @@ not your fork. If you are posting a review request, put the pull request number
field. Then, when you close the request, you can navigate from the bug number to easily close
the pull request.

If your CI-build failed in Travis-CI, and the failure looks like it's not due to
your change, please open an issue with the part of the CI log containing the test failure and label
the issue with `flaky-test`. If an issue already exists, add a comment to it noting that you
encountered it too. After you've done that, you can ask in slack for someone to restart the shard.
If your CI-build failed in Travis-CI, and the failure looks like it's not due to your change, please
go to https://github.com/pantsbuild/pants/issues/9386 and follow the instructions there to deal with
the misbehaving test. After you've done that, you can ask in slack for someone to restart the shard.
That will cause the shard to re-run its tests.

#### Advanced: Remote execution of tests
Expand Down
3 changes: 2 additions & 1 deletion src/rust/engine/process_execution/src/remote_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1145,7 +1145,7 @@ async fn timeout_after_sufficiently_delayed_getoperations() {
assert_cancellation_requests(&mock_server, vec![op_name.to_owned()]);
}

#[ignore] // https://github.com/pantsbuild/pants/issues/8405
#[ignore] // flaky: https://github.com/pantsbuild/pants/issues/8405
#[tokio::test]
async fn dropped_request_cancels() {
let request_timeout = Duration::new(10, 0);
Expand Down Expand Up @@ -1229,6 +1229,7 @@ async fn dropped_request_cancels() {
assert_cancellation_requests(&mock_server, vec![op_name.to_owned()]);
}

#[ignore] // flaky: https://github.com/pantsbuild/pants/issues/7149
#[tokio::test]
async fn retry_for_cancelled_channel() {
let execute_request = echo_foo_request();
Expand Down
1 change: 1 addition & 0 deletions src/rust/engine/serverset/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ async fn reattempts_unhealthy() {
expect_both(&s, 3).await
}

#[ignore] // flaky: https://github.com/pantsbuild/pants/issues/7836
#[tokio::test]
async fn backoff_when_unhealthy() {
let s = Serverset::new(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ def test_list_descendants(self):
"a/b:b", "a/b/c:c", "a/b/c:c2", "a/b/c:c3", "a/b/d:d", "a/b/e:e1", args=["a/b::"]
)

@pytest.mark.flaky(retries=1) # https://github.com/pantsbuild/pants/issues/8678
@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/8678")
def test_list_all(self):
self.assert_entries(
"\n",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
from collections import namedtuple
from contextlib import contextmanager

import pytest

from pants.testutil.pants_run_integration_test import PantsRunIntegrationTest
from pants.util.contextutil import temporary_dir

Expand Down Expand Up @@ -198,6 +200,7 @@ def test_unit_tests_with_different_sets(self):
self.assert_failure(run)
self.assertIn("Undefined revs for jars unsupported by Coursier", run.stdout_data)

@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/9313")
def test_unit_tests_with_different_sets_one_batch(self):
run = self.run_pants(
[
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
from textwrap import dedent
from typing import Callable, Dict, List

import pytest

from pants.backend.jvm.tasks.jvm_compile.rsc.rsc_compile import RscCompile
from pants.base.build_environment import get_buildroot
from pants.fs.archive import TGZ, ZIP
Expand Down Expand Up @@ -290,6 +292,7 @@ def config(incremental_caching):
Compile({srcfile: "public final class A {}"}, config(True), 2),
)

@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/9312")
def test_rsc_and_zinc_caching(self):
"""Tests that with rsc-and-zinc, we write both artifacts."""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ def test_java_compile_with_different_resolved_jars_produce_different_artifacts(s
self.assertEqual(guava_16_artifact_dir, guava_15_artifact_dir)
self.assertEqual(len(os.listdir(guava_16_artifact_dir)), 2)

@pytest.mark.flaky(retries=1) # https://github.com/pantsbuild/pants/issues/8171
@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/8171")
def test_java_compile_with_corrupt_remote(self):
"""Tests that a corrupt artifact in the remote cache still results in a successful
compile."""
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import pytest

from pants_test.backend.jvm.tasks.jvm_compile.rsc.rsc_compile_integration_base import (
RscCompileIntegrationBase,
ensure_compile_rsc_execution_strategy,
Expand All @@ -24,6 +26,7 @@ def test_executing_multi_target_binary(self):
pants_run = self.do_command("run", "examples/src/scala/org/pantsbuild/example/hello/exe")
self.assertIn("Hello, Resource World!", pants_run.stdout_data)

@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/8679")
@ensure_compile_rsc_execution_strategy(RscCompileIntegrationBase.rsc_and_zinc)
def test_java_with_transitive_exported_scala_dep(self):
self.do_command(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ class RscCompileIntegrationYoutline(RscCompileIntegrationBase):
def test_basic_binary(self):
self._testproject_compile("mutual", "bin", "A")

@pytest.mark.flaky(retries=1)
@pytest.mark.skip(reason="flaky")
@ensure_compile_rsc_execution_strategy(RscCompileIntegrationBase.outline_and_zinc)
def test_public_inference(self):
self._testproject_compile("public_inference", "public_inference", "PublicInference")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import pytest

from pants.testutil.pants_run_integration_test import PantsRunIntegrationTest


Expand All @@ -11,6 +13,7 @@ class BackendIndependenceTest(PantsRunIntegrationTest):
def hermetic(cls):
return True

@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/7168")
def test_independent_test_run(self):
pants_run = self.run_pants(
command=["test", "examples/tests/java/org/pantsbuild/example/hello/greet"],
Expand Down
8 changes: 4 additions & 4 deletions tests/python/pants_test/cache/test_artifact_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,15 +99,15 @@ def test_local_cache_with_seperate_extraction_root(self):
with self.setup_local_cache(seperate_extraction_root=True) as artifact_cache:
self.do_test_artifact_cache(artifact_cache)

@pytest.mark.flaky(retries=1) # https://github.com/pantsbuild/pants/issues/6838
@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6838")
def test_restful_cache(self):
with self.assertRaises(InvalidRESTfulCacheProtoError):
RESTfulArtifactCache("foo", BestUrlSelector(["ftp://localhost/bar"]), "foo")

with self.setup_rest_cache() as artifact_cache:
self.do_test_artifact_cache(artifact_cache)

@pytest.mark.flaky(retries=1) # https://github.com/pantsbuild/pants/issues/6838
@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6838")
def test_restful_cache_failover(self):
bad_url = "http://badhost:123"

Expand Down Expand Up @@ -239,7 +239,7 @@ def test_local_backed_remote_cache_corrupt_artifact(self):
self.assertTrue(os.path.exists(results_dir))
self.assertTrue(len(os.listdir(results_dir)) == 0)

@pytest.mark.flaky(retries=1) # https://github.com/pantsbuild/pants/issues/6838
@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6838")
def test_multiproc(self):
key = CacheKey("muppet_key", "fake_hash")

Expand All @@ -265,7 +265,7 @@ def test_failed_multiproc(self):
call_insert((cache, key, [path], False))
self.assertFalse(call_use_cached_files((cache, key, None)))

@pytest.mark.flaky(retries=1) # https://github.com/pantsbuild/pants/issues/6838
@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6838")
def test_noops_after_max_retries_exceeded(self):
key = CacheKey("muppet_key", "fake_hash")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ def setup_bundle_target(self) -> Iterator[None]:
with self.with_overwritten_file_content(build_path, f"{original_content}\n{new_content}"):
yield

@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/8520")
def test_missing_sources_warnings(self):
target_to_unmatched_globs = {
"missing-globs": ["*.a"],
Expand Down Expand Up @@ -144,6 +145,7 @@ def test_missing_sources_warnings(self):
in pants_run.stderr_data
)

@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/8520")
def test_existing_sources(self):
target_full = f"{self._SOURCES_TARGET_BASE}:text"
pants_run = self.run_pants(
Expand Down Expand Up @@ -178,6 +180,7 @@ def test_missing_bundles_warnings(self):
in pants_run.stderr_data
)

@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/8520")
def test_existing_bundles(self):
target_full = f"{self._BUNDLE_TARGET_BASE}:mapper"
pants_run = self.run_pants(
Expand All @@ -192,7 +195,7 @@ def test_existing_directory_with_no_build_files_fails(self):
self.assert_failure(pants_run)
self.assertIn("does not match any targets.", pants_run.stderr_data)

@unittest.skip("Flaky: https://github.com/pantsbuild/pants/issues/6787")
@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/6787")
def test_error_message(self):
with self.setup_bundle_target(), self.setup_sources_targets():
for target in self._ERR_TARGETS:
Expand Down
4 changes: 1 addition & 3 deletions tests/python/pants_test/engine/test_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,9 +223,7 @@ def fork_context_body():
res = self.mk_scheduler().with_fork_context(fork_context_body)
self.assertEquals(res, expected)

@unittest.skip(
"Inherently flaky as described in https://github.com/pantsbuild/pants/issues/6829"
)
@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/6829")
def test_trace_multi(self):
# Tests that when multiple distinct failures occur, they are each rendered.

Expand Down
16 changes: 8 additions & 8 deletions tests/python/pants_test/pantsd/test_pantsd_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def test_pantsd_compile(self):
pantsd_run(["compile", "examples/src/scala/org/pantsbuild/example/hello/welcome"])
checker.assert_started()

@unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7573")
@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7573")
def test_pantsd_run(self):
with self.pantsd_successful_run_context("debug") as (
pantsd_run,
Expand Down Expand Up @@ -124,7 +124,7 @@ def test_pantsd_pantsd_runner_doesnt_die_after_failed_run(self):
)
checker.assert_running()

@pytest.mark.flaky(retries=1) # https://github.com/pantsbuild/pants/issues/6114
@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6114")
def test_pantsd_lifecycle_invalidation(self):
"""Runs pants commands with pantsd enabled, in a loop, alternating between options that
should invalidate pantsd and incur a restart and then asserts for pid consistency."""
Expand Down Expand Up @@ -216,7 +216,7 @@ def test_pantsd_aligned_output(self):
for run_pairs in zip(non_daemon_runs, daemon_runs):
self.assertEqual(*(run.stdout_data for run in run_pairs))

@unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7622")
@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7622")
def test_pantsd_filesystem_invalidation(self):
"""Runs with pantsd enabled, in a loop, while another thread invalidates files."""
with self.pantsd_successful_run_context() as (pantsd_run, checker, workdir, _):
Expand Down Expand Up @@ -385,7 +385,7 @@ def test_pantsd_pid_change(self):
# Remove the pidfile so that the teardown script doesn't try to kill process 9.
os.unlink(pidpath)

@pytest.mark.flaky(retries=1) # https://github.com/pantsbuild/pants/issues/8193
@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/8193")
def test_pantsd_memory_usage(self):
"""Validates that after N runs, memory usage has increased by no more than X percent."""
number_of_runs = 10
Expand Down Expand Up @@ -542,7 +542,7 @@ def _assert_pantsd_keyboardinterrupt_signal(self, signum, regexps=[], quit_timeo
# The pantsd processes should be dead, and they should have exited with 1.
self.assertFalse(proc.is_running())

@unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7554")
@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7554")
def test_pantsd_sigterm(self):
self._assert_pantsd_keyboardinterrupt_signal(
signal.SIGTERM,
Expand All @@ -560,7 +560,7 @@ def test_pantsd_sigterm(self):
],
)

@unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7572")
@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7572")
def test_pantsd_sigquit(self):
self._assert_pantsd_keyboardinterrupt_signal(
signal.SIGQUIT,
Expand All @@ -578,7 +578,7 @@ def test_pantsd_sigquit(self):
],
)

@unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7547")
@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7547")
def test_pantsd_sigint(self):
self._assert_pantsd_keyboardinterrupt_signal(
signal.SIGINT,
Expand All @@ -592,7 +592,7 @@ def test_pantsd_sigint(self):
],
)

@unittest.skip("Flaky as described in: https://github.com/pantsbuild/pants/issues/7457")
@unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7457")
def test_signal_pailgun_stream_timeout(self):
# NB: The actual timestamp has the date and time at sub-second granularity. The date is just
# used here since that is known in advance in order to assert that the timestamp is well-formed.
Expand Down
2 changes: 1 addition & 1 deletion tests/python/pants_test/pantsd/test_process_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ def test_readwrite_metadata_by_name(self):
self.pmm.read_metadata_by_name(self.NAME, self.TEST_KEY, int), self.TEST_VALUE_INT
)

@pytest.mark.flaky(retries=1) # https://github.com/pantsbuild/pants/issues/6836
@pytest.mark.skip(reason="flaky: https://github.com/pantsbuild/pants/issues/6836")
def test_deadline_until(self):
with self.assertRaises(ProcessMetadataManager.Timeout):
with self.captured_logging(logging.INFO) as captured:
Expand Down

0 comments on commit ec40520

Please sign in to comment.