diff --git a/services/test_results.py b/services/test_results.py index 13fe74e7b..73b66db3f 100644 --- a/services/test_results.py +++ b/services/test_results.py @@ -157,8 +157,28 @@ def make_quoted(content: str) -> str: return f"\n{result}\n" +def properly_backtick(content: str) -> str: + max_backtick_count = 0 + curr_backtick_count = 0 + prev_char = None + for char in content: + if char == "`": + curr_backtick_count += 1 + else: + curr_backtick_count = 0 + + if curr_backtick_count > max_backtick_count: + max_backtick_count = curr_backtick_count + + backticks = "`" * (max_backtick_count + 1) + return f"{backticks}\n{content}\n{backticks}" + + def wrap_in_code(content: str) -> str: - return f"
\n{content}\n
" + if "```" in content: + return properly_backtick(content) + else: + return f"\n```\n{content}\n```\n" def display_duration(f: float) -> str: @@ -177,10 +197,11 @@ def generate_failure_info( else: failure_message = "No failure message available" + failure_message = wrap_in_code(failure_message) if fail.build_url: - return f"
{failure_message}
\n[View]({fail.build_url}) the CI Build" + return f"{failure_message}\n[View]({fail.build_url}) the CI Build" else: - return f"
{failure_message}
" + return failure_message def generate_view_test_analytics_line(commit: Commit) -> str: @@ -192,7 +213,7 @@ def generate_view_test_analytics_line(commit: Commit) -> str: def messagify_failure( failure: TestResultsNotificationFailure, ) -> str: - test_name = wrap_in_code(failure.testname) + test_name = wrap_in_code(failure.testname.replace("\x1f", " ")) formatted_duration = display_duration(failure.duration_seconds) stack_trace_summary = f"Stack Traces | {formatted_duration}s run time" stack_trace = wrap_in_details( @@ -206,7 +227,7 @@ def messagify_flake( flaky_failure: TestResultsNotificationFailure, flake_info: FlakeInfo, ) -> str: - test_name = wrap_in_code(flaky_failure.testname) + test_name = wrap_in_code(flaky_failure.testname.replace("\x1f", " ")) formatted_duration = display_duration(flaky_failure.duration_seconds) flake_rate = flake_info.failed / flake_info.count * 100 flake_rate_section = f"**Flake rate in main:** {flake_rate}% (Passed {flake_info.count - flake_info.failed} times, Failed {flake_info.failed} times)" diff --git a/services/tests/test_test_results.py b/services/tests/test_test_results.py index 7ee9dd3e5..e89bbbc4b 100644 --- a/services/tests/test_test_results.py +++ b/services/tests/test_test_results.py @@ -84,7 +84,12 @@ def test_generate_failure_info(): assert ( res - == "
hello world
\n[View](https://example.com/build_url) the CI Build" + == """ +``` +hello world +``` + +[View](https://example.com/build_url) the CI Build""" ) @@ -113,12 +118,18 @@ def test_build_message(): | 3 | 1 | 2 | 3 |
View the top 1 failed tests by shortest run time ->
+> 
+> ```
 > testname
-> 
+> ``` +> >
Stack Traces | 1s run time > -> >
hello world
+> > +> > ``` +> > hello world +> > ``` +> > > > [View](https://example.com/build_url) the CI Build > >
@@ -157,13 +168,19 @@ def test_build_message_with_flake(): | 3 | 1 | 2 | 3 |
View the full list of 1 :snowflake: flaky tests ->
+> 
+> ```
 > testname
-> 
+> ``` +> > **Flake rate in main:** 33.33333333333333% (Passed 2 times, Failed 1 times) >
Stack Traces | 1s run time > -> >
hello world
+> > +> > ``` +> > hello world +> > ``` +> > > > [View](https://example.com/build_url) the CI Build > >
diff --git a/tasks/test_results_finisher.py b/tasks/test_results_finisher.py index 1fa95df7e..024d76cec 100644 --- a/tasks/test_results_finisher.py +++ b/tasks/test_results_finisher.py @@ -34,13 +34,7 @@ test_results_finisher_task_name = "app.tasks.test_results.TestResultsFinisherTask" ESCAPE_FAILURE_MESSAGE_DEFN = [ - Replacement(['"'], """, EscapeEnum.REPLACE), - Replacement(["'"], "'", EscapeEnum.REPLACE), - Replacement(["<"], "<", EscapeEnum.REPLACE), - Replacement([">"], ">", EscapeEnum.REPLACE), - Replacement(["?"], "&", EscapeEnum.REPLACE), Replacement(["\r"], "", EscapeEnum.REPLACE), - Replacement(["\n"], "
", EscapeEnum.REPLACE), ] QUEUE_NOTIFY_KEY = "queue_notify" diff --git a/tasks/tests/unit/test_test_results_finisher.py b/tasks/tests/unit/test_test_results_finisher.py index 875b82b5b..6e098c3bc 100644 --- a/tasks/tests/unit/test_test_results_finisher.py +++ b/tasks/tests/unit/test_test_results_finisher.py @@ -70,9 +70,9 @@ def test_results_setup(mocker, dbsession): message="hello world", commitid="cd76b0821854a780b60012aed85af0a8263004ad", repository__owner__unencrypted_oauth_token="test7lk5ndmtqzxlx06rip65nac9c7epqopclnoy", - repository__owner__username="joseph-sentry", + repository__owner__username="test-username", repository__owner__service="github", - repository__name="codecov-demo", + repository__name="test-repo-name", ) commit.branch = "main" dbsession.add(commit) @@ -176,14 +176,14 @@ def test_results_setup(mocker, dbsession): TestInstance( test_id=test_id2, outcome=str(Outcome.Failure), - failure_message="Shared failure message", + failure_message="Shared \n\n\n\n
 ````````\n \r\n\r\n | test | test | test 
failure message", duration_seconds=2.0, upload_id=uploads[1].id, ), TestInstance( test_id=test_id3, outcome=str(Outcome.Failure), - failure_message="Shared failure message", + failure_message="Shared \n\n\n\n
 \n  ````````  \n \r\n\r\n | test | test | test 
failure message", duration_seconds=3.0, upload_id=uploads[2].id, ), @@ -351,7 +351,78 @@ def test_upload_finisher_task_call( assert expected_result == result mock_repo_provider_comments.post_comment.assert_called_with( pull.pullid, - "### :x: 4 Tests Failed:\n| Tests completed | Failed | Passed | Skipped |\n|---|---|---|---|\n| 4 | 4 | 0 | 0 |\n
View the top 3 failed tests by shortest run time\n\n>
\n> test_name1\n> 
\n>
Stack Traces | 2s run time\n> \n> >
Shared failure message
\n> > [View](https://example.com/build_url_1) the CI Build\n> \n>
\n\n\n>
\n> Other Class Name\x1ftest_name2\n> 
\n>
Stack Traces | 3s run time\n> \n> >
Shared failure message
\n> > [View](https://example.com/build_url_2) the CI Build\n> \n>
\n\n\n>
\n> Class Name\x1ftest_name0\n> 
\n>
Stack Traces | 4s run time\n> \n> >
<pre>Fourth 

</pre> | test | instance |
\n> > [View](https://example.com/build_url_3) the CI Build\n> \n>
\n\n
\n\nTo view individual test run time comparison to the main branch, go to the [Test Analytics Dashboard](https://app.codecov.io/gh/joseph-sentry/codecov-demo/tests/main)", + """### :x: 4 Tests Failed: +| Tests completed | Failed | Passed | Skipped | +|---|---|---|---| +| 4 | 4 | 0 | 0 | +
View the top 3 failed tests by shortest run time + +> +> ``` +> test_name1 +> ``` +> +>
Stack Traces | 2s run time +> +> > ````````` +> > Shared +> > +> > +> > +> >
 ````````
+> >  
+> > 
+> >  | test | test | test 
failure message +> > ````````` +> > [View](https://example.com/build_url_1) the CI Build +> +>
+ + +> +> ``` +> Other Class Name test_name2 +> ``` +> +>
Stack Traces | 3s run time +> +> > ````````` +> > Shared +> > +> > +> > +> >
 
+> >   ````````  
+> >  
+> > 
+> >  | test | test | test 
failure message +> > ````````` +> > [View](https://example.com/build_url_2) the CI Build +> +>
+ + +> +> ``` +> Class Name test_name0 +> ``` +> +>
Stack Traces | 4s run time +> +> > +> > ``` +> >
Fourth 
+> > 
+> > 
| test | instance | +> > ``` +> > +> > [View](https://example.com/build_url_3) the CI Build +> +>
+ +
+ +To view individual test run time comparison to the main branch, go to the [Test Analytics Dashboard](https://app.codecov.io/gh/test-username/test-repo-name/tests/main)""", ) mock_metrics.incr.assert_has_calls( @@ -551,7 +622,78 @@ def test_upload_finisher_task_call_existing_comment( mock_repo_provider_comments.edit_comment.assert_called_with( pull.pullid, 1, - "### :x: 4 Tests Failed:\n| Tests completed | Failed | Passed | Skipped |\n|---|---|---|---|\n| 4 | 4 | 0 | 0 |\n
View the top 3 failed tests by shortest run time\n\n>
\n> test_name1\n> 
\n>
Stack Traces | 2s run time\n> \n> >
Shared failure message
\n> > [View](https://example.com/build_url_1) the CI Build\n> \n>
\n\n\n>
\n> Other Class Name\x1ftest_name2\n> 
\n>
Stack Traces | 3s run time\n> \n> >
Shared failure message
\n> > [View](https://example.com/build_url_2) the CI Build\n> \n>
\n\n\n>
\n> Class Name\x1ftest_name0\n> 
\n>
Stack Traces | 4s run time\n> \n> >
<pre>Fourth 

</pre> | test | instance |
\n> > [View](https://example.com/build_url_3) the CI Build\n> \n>
\n\n
\n\nTo view individual test run time comparison to the main branch, go to the [Test Analytics Dashboard](https://app.codecov.io/gh/joseph-sentry/codecov-demo/tests/main)", + """### :x: 4 Tests Failed: +| Tests completed | Failed | Passed | Skipped | +|---|---|---|---| +| 4 | 4 | 0 | 0 | +
View the top 3 failed tests by shortest run time + +> +> ``` +> test_name1 +> ``` +> +>
Stack Traces | 2s run time +> +> > ````````` +> > Shared +> > +> > +> > +> >
 ````````
+> >  
+> > 
+> >  | test | test | test 
failure message +> > ````````` +> > [View](https://example.com/build_url_1) the CI Build +> +>
+ + +> +> ``` +> Other Class Name test_name2 +> ``` +> +>
Stack Traces | 3s run time +> +> > ````````` +> > Shared +> > +> > +> > +> >
 
+> >   ````````  
+> >  
+> > 
+> >  | test | test | test 
failure message +> > ````````` +> > [View](https://example.com/build_url_2) the CI Build +> +>
+ + +> +> ``` +> Class Name test_name0 +> ``` +> +>
Stack Traces | 4s run time +> +> > +> > ``` +> >
Fourth 
+> > 
+> > 
| test | instance | +> > ``` +> > +> > [View](https://example.com/build_url_3) the CI Build +> +>
+ +
+ +To view individual test run time comparison to the main branch, go to the [Test Analytics Dashboard](https://app.codecov.io/gh/test-username/test-repo-name/tests/main)""", ) assert expected_result == result @@ -677,7 +819,81 @@ def test_upload_finisher_task_call_with_flaky( mock_repo_provider_comments.post_comment.assert_called_with( pull.pullid, - "### :x: 4 Tests Failed:\n| Tests completed | Failed | Passed | Skipped |\n|---|---|---|---|\n| 4 | 4 | 0 | 0 |\n
View the top 2 failed tests by shortest run time\n\n>
\n> test_name1\n> 
\n>
Stack Traces | 2s run time\n> \n> >
Shared failure message
\n> > [View](https://example.com/build_url_1) the CI Build\n> \n>
\n\n\n>
\n> Class Name\x1ftest_name0\n> 
\n>
Stack Traces | 4s run time\n> \n> >
<pre>Fourth 

</pre> | test | instance |
\n> > [View](https://example.com/build_url_3) the CI Build\n> \n>
\n\n
\n
View the full list of 1 :snowflake: flaky tests\n\n>
\n> Other Class Name\x1ftest_name2\n> 
\n> **Flake rate in main:** 40.0% (Passed 3 times, Failed 2 times)\n>
Stack Traces | 3s run time\n> \n> >
Shared failure message
\n> > [View](https://example.com/build_url_2) the CI Build\n> \n>
\n\n
\n\nTo view individual test run time comparison to the main branch, go to the [Test Analytics Dashboard](https://app.codecov.io/gh/joseph-sentry/codecov-demo/tests/main)", + """### :x: 4 Tests Failed: +| Tests completed | Failed | Passed | Skipped | +|---|---|---|---| +| 4 | 4 | 0 | 0 | +
View the top 2 failed tests by shortest run time + +> +> ``` +> test_name1 +> ``` +> +>
Stack Traces | 2s run time +> +> > ````````` +> > Shared +> > +> > +> > +> >
 ````````
+> >  
+> > 
+> >  | test | test | test 
failure message +> > ````````` +> > [View](https://example.com/build_url_1) the CI Build +> +>
+ + +> +> ``` +> Class Name test_name0 +> ``` +> +>
Stack Traces | 4s run time +> +> > +> > ``` +> >
Fourth 
+> > 
+> > 
| test | instance | +> > ``` +> > +> > [View](https://example.com/build_url_3) the CI Build +> +>
+ +
+
View the full list of 1 :snowflake: flaky tests + +> +> ``` +> Other Class Name test_name2 +> ``` +> +> **Flake rate in main:** 40.0% (Passed 3 times, Failed 2 times) +>
Stack Traces | 3s run time +> +> > ````````` +> > Shared +> > +> > +> > +> >
 
+> >   ````````  
+> >  
+> > 
+> >  | test | test | test 
failure message +> > ````````` +> > [View](https://example.com/build_url_2) the CI Build +> +>
+ +
+ +To view individual test run time comparison to the main branch, go to the [Test Analytics Dashboard](https://app.codecov.io/gh/test-username/test-repo-name/tests/main)""", ) mock_metrics.incr.assert_has_calls(