diff --git a/tasks/test_results_finisher.py b/tasks/test_results_finisher.py index c2f0bea9c..bff245192 100644 --- a/tasks/test_results_finisher.py +++ b/tasks/test_results_finisher.py @@ -399,6 +399,7 @@ def get_flaky_tests( Flake.repoid == repoid, Flake.testid.in_(failure_test_ids), Flake.end_date.is_(None), + Flake.count != (Flake.recent_passes_count + Flake.fail_count), ) .limit(100) .all() diff --git a/tasks/tests/unit/test_test_results_finisher.py b/tasks/tests/unit/test_test_results_finisher.py index 42ffd7331..372778447 100644 --- a/tasks/tests/unit/test_test_results_finisher.py +++ b/tasks/tests/unit/test_test_results_finisher.py @@ -763,6 +763,9 @@ def test_upload_finisher_task_call_comment_fails( assert expected_result == result + @pytest.mark.parametrize( + "fail_count,count,recent_passes_count", [(2, 15, 13), (50, 150, 10)] + ) @pytest.mark.integration def test_upload_finisher_task_call_with_flaky( self, @@ -776,12 +779,20 @@ def test_upload_finisher_task_call_with_flaky( test_results_mock_app, mock_repo_provider_comments, test_results_setup, + fail_count, + count, + recent_passes_count, ): mock_feature = mocker.patch("services.test_results.FLAKY_TEST_DETECTION") mock_feature.check_value.return_value = True repoid, commit, pull, test_instances = test_results_setup + for i, instance in enumerate(test_instances): + if i != 2: + dbsession.delete(instance) + dbsession.flush() + r = ReducedError() r.message = "failure_message" @@ -792,9 +803,9 @@ def test_upload_finisher_task_call_with_flaky( f.repoid = repoid f.testid = test_instances[2].test_id f.reduced_error = r - f.count = 5 - f.fail_count = 2 - f.recent_passes_count = 1 + f.count = count + f.fail_count = fail_count + f.recent_passes_count = recent_passes_count f.start_date = datetime.now() f.end_date = None @@ -824,61 +835,17 @@ def test_upload_finisher_task_call_with_flaky( mock_repo_provider_comments.post_comment.assert_called_with( pull.pullid, - """### :x: 4 Tests Failed: + f"""### :x: 1 Tests Failed: | Tests completed | Failed | Passed | Skipped | |---|---|---|---| -| 4 | 4 | 0 | 0 | -
View the top 2 failed tests by shortest run time - -> -> ``` -> test_name1 -> ``` -> ->
Stack Traces | 2s run time -> -> > ````````` -> > Shared -> > -> > -> > -> >
 ````````
-> >  
-> > 
-> >  | test | test | test 
failure message -> > ````````` -> > [View](https://example.com/build_url_1) the CI Build -> ->
- - -> -> ``` -> Class Name test_name0 -> ``` -> ->
Stack Traces | 4s run time -> -> > -> > ``` -> >
Fourth 
-> > 
-> > 
| test | instance | -> > ``` -> > -> > [View](https://example.com/build_url_3) the CI Build -> ->
- -
-
View the full list of 1 :snowflake: flaky tests +| 1 | 1 | 0 | 0 | +
{"View the top 1 failed tests by shortest run time" if (count - fail_count) == recent_passes_count else "View the full list of 1 :snowflake: flaky tests"} > > ``` > Other Class Name test_name2 > ``` -> -> **Flake rate in main:** 40.00% (Passed 3 times, Failed 2 times) +> {f"\n> **Flake rate in main:** 33.33% (Passed {count - fail_count} times, Failed {fail_count} times)" if (count - fail_count) != recent_passes_count else ""} >
Stack Traces | 3s run time > > > `````````