Skip to content

Commit

Permalink
feat: update test_results_parser version and start computing names
Browse files Browse the repository at this point in the history
This commit updates the version of the test_results_parser library
that we are using:
- no more support for parsing vitest.json or pytest-reportlog files
- parse_junit_xml now returns a ParsingInfo class, the type signature
  looks something like:

    class ParsingInfo:
        testruns: list[Testrun]
        framework: framework

  where Framework is:

    class Framework(Enum):
        Pytest
        Jest
        Vitest
        PHPUnit

- Testrun also now contains the filename field which is an optional
  string and a classname field

We've also started computing the name of the test which just means
depending on the framework we've detected we transform the raw name
and classname from the JUnit file to something more readable and if
that computed name is available it's what we should show in the PR
comment
  • Loading branch information
joseph-sentry committed Sep 25, 2024
1 parent b94e31d commit cf09fe4
Show file tree
Hide file tree
Showing 13 changed files with 432 additions and 392 deletions.
5 changes: 5 additions & 0 deletions database/models/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,11 @@ def id(self):
# for example: the same test being run on windows vs. mac
flags_hash = Column(types.String(256), nullable=False)

framework = Column(types.String(100), nullable=True)

computed_name = Column(types.Text, nullable=True)
filename = Column(types.Text, nullable=True)

__table_args__ = (
UniqueConstraint(
"repoid",
Expand Down
2 changes: 1 addition & 1 deletion requirements.in
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem
https://github.com/codecov/shared/archive/106b0ae2b9a2870899fa3903fc6da0a9ba67eef2.tar.gz#egg=shared
https://github.com/codecov/test-results-parser/archive/1507de2241601d678e514c08b38426e48bb6d47d.tar.gz#egg=test-results-parser
https://github.com/codecov/test-results-parser/archive/ef39a0888acd62d02a316a852a15d755c74e78c6.tar.gz#egg=test-results-parser
https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2f887936a98a668dda42.tar.gz#egg=timestring
asgiref>=3.7.2
analytics-python==1.3.0b1
Expand Down
4 changes: 2 additions & 2 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile with Python 3.12
# by the following command:
#
# pip-compile
# pip-compile requirements.in
#
amqp==5.2.0
# via kombu
Expand Down Expand Up @@ -391,7 +391,7 @@ statsd==3.3.0
# shared
stripe==9.6.0
# via -r requirements.in
test-results-parser @ https://github.com/codecov/test-results-parser/archive/1507de2241601d678e514c08b38426e48bb6d47d.tar.gz
test-results-parser @ https://github.com/codecov/test-results-parser/archive/ef39a0888acd62d02a316a852a15d755c74e78c6.tar.gz
# via -r requirements.in
text-unidecode==1.3
# via faker
Expand Down
9 changes: 4 additions & 5 deletions services/test_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,7 @@ def generate_test_id(repoid, testsuite, name, flags_hash):
@dataclass
class TestResultsNotificationFailure:
failure_message: str
testsuite: str
testname: str
display_name: str
envs: List[str]
test_id: str
duration_seconds: float
Expand Down Expand Up @@ -205,7 +204,7 @@ def generate_view_test_analytics_line(commit: Commit) -> str:
def messagify_failure(
failure: TestResultsNotificationFailure,
) -> str:
test_name = wrap_in_code(failure.testname.replace("\x1f", " "))
test_name = wrap_in_code(failure.display_name.replace("\x1f", " "))
formatted_duration = display_duration(failure.duration_seconds)
stack_trace_summary = f"Stack Traces | {formatted_duration}s run time"
stack_trace = wrap_in_details(
Expand All @@ -219,7 +218,7 @@ def messagify_flake(
flaky_failure: TestResultsNotificationFailure,
flake_info: FlakeInfo,
) -> str:
test_name = wrap_in_code(flaky_failure.testname.replace("\x1f", " "))
test_name = wrap_in_code(flaky_failure.display_name.replace("\x1f", " "))
formatted_duration = display_duration(flaky_failure.duration_seconds)
flake_rate = flake_info.failed / flake_info.count * 100
flake_rate_section = f"**Flake rate in main:** {flake_rate:.2f}% (Passed {flake_info.count - flake_info.failed} times, Failed {flake_info.failed} times)"
Expand Down Expand Up @@ -256,7 +255,7 @@ def build_message(self) -> str:
lambda x: x.test_id not in self.payload.flaky_tests,
self.payload.failures,
),
key=lambda x: (x.duration_seconds, x.testname),
key=lambda x: (x.duration_seconds, x.display_name),
)

if failures:
Expand Down
3 changes: 0 additions & 3 deletions services/tests/test_test_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ def test_generate_failure_info():
test_id = generate_test_id(1, "testsuite", "testname", flags_hash)
fail = TestResultsNotificationFailure(
"hello world",
"testsuite",
"testname",
[],
test_id,
Expand All @@ -100,7 +99,6 @@ def test_build_message():
test_id = generate_test_id(1, "testsuite", "testname", flags_hash)
fail = TestResultsNotificationFailure(
"hello world",
"testsuite",
"testname",
[],
test_id,
Expand Down Expand Up @@ -147,7 +145,6 @@ def test_build_message_with_flake():
test_id = generate_test_id(1, "testsuite", "testname", flags_hash)
fail = TestResultsNotificationFailure(
"hello world",
"testsuite",
"testname",
[],
test_id,
Expand Down
5 changes: 3 additions & 2 deletions tasks/test_results_finisher.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,8 +232,9 @@ def process_impl_within_lock(

failures.append(
TestResultsNotificationFailure(
testsuite=test_instance.test.testsuite,
testname=test_instance.test.name,
display_name=test_instance.test.computed_name
if test_instance.test.computed_name is not None
else test_instance.test.name,
failure_message=failure_message,
test_id=test_instance.test_id,
envs=flag_names,
Expand Down
Loading

0 comments on commit cf09fe4

Please sign in to comment.