-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathtest_custom_report.py
224 lines (188 loc) · 10.2 KB
/
test_custom_report.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
# coding=utf-8
from __future__ import unicode_literals
import pytest
import custom_report
def test_pass_custom_symbol(testdir, one_test_passing):
result = testdir.runpytest('--report-passed=👍')
result.stdout.fnmatch_lines([
'test_pass_custom_symbol.py 👍 *',
])
assert result.ret == 0
result.assert_outcomes(passed=1, skipped=0, failed=0, error=0)
def test_pass_custom_symbol_verbose(testdir, one_test_passing):
result = testdir.runpytest('--report-passed-verbose=OHYEAH', '-v')
result.stdout.fnmatch_lines([
'test_pass_custom_symbol_verbose.py::test_passing OHYEAH *',
])
assert result.ret == 0
result.assert_outcomes(passed=1, skipped=0, failed=0, error=0)
def test_fail_custom_symbol(testdir, two_tests_failing):
result = testdir.runpytest('--report-failed=💩')
result.stdout.fnmatch_lines([
'test_fail_custom_symbol.py 💩💩 *',
])
assert result.ret == 1
result.assert_outcomes(passed=0, skipped=0, failed=2, error=0)
def test_fail_custom_symbol_verbose(testdir, two_tests_failing):
result = testdir.runpytest('--report-failed-verbose=OHCRAP', '-v')
result.stdout.fnmatch_lines([
'test_fail_custom_symbol_verbose.py::test_failing OHCRAP *',
'test_fail_custom_symbol_verbose.py::test_raises OHCRAP *',
])
assert result.ret == 1
result.assert_outcomes(passed=0, skipped=0, failed=2, error=0)
def test_skip_custom_symbol(testdir, bunch_of_skipped_tests):
result = testdir.runpytest('--report-skipped=?')
result.stdout.fnmatch_lines([
'test_skip_custom_symbol.py ????.??? *',
])
assert result.ret == 0
result.assert_outcomes(passed=1, skipped=7, failed=0, error=0)
def test_skip_custom_symbol_verbose(testdir, bunch_of_skipped_tests):
result = testdir.runpytest('--report-skipped-verbose=WHATEVS', '--verbose')
result.stdout.fnmatch_lines([
'test_skip_custom_symbol_verbose.py::test_skip WHATEVS *',
'test_skip_custom_symbol_verbose.py::test_error_in_setup_and_test_is_skipped WHATEVS *',
'test_skip_custom_symbol_verbose.py::test_error_in_teardown_and_test_is_skipped WHATEVS *',
'test_skip_custom_symbol_verbose.py::test_skipif_true WHATEVS *',
'test_skip_custom_symbol_verbose.py::test_skipif_false PASSED *',
'test_skip_custom_symbol_verbose.py::test_skip_and_xfail WHATEVS *',
'test_skip_custom_symbol_verbose.py::test_xfail_and_skip WHATEVS *',
'test_skip_custom_symbol_verbose.py::test_skipped_within_test WHATEVS *',
])
assert result.ret == 0
result.assert_outcomes(passed=1, skipped=7, failed=0, error=0)
def test_error_in_setup_and_teardown(testdir, errors_in_setup_and_teardown):
result = testdir.runpytest('--report-error=!')
result.stdout.fnmatch_lines([
'test_error_in_setup_and_teardown.py !.!! *',
])
assert result.ret == 1
result.assert_outcomes(passed=1, skipped=0, failed=0, error=3)
def test_error_in_setup_and_teardown_verbose(testdir, errors_in_setup_and_teardown):
result = testdir.runpytest('--report-error-verbose=UHOH', '--verbose')
result.stdout.fnmatch_lines([
'test_error_in_setup_and_teardown_verbose.py::test_error_in_setup UHOH *',
'test_error_in_setup_and_teardown_verbose.py::test_error_in_teardown PASSED *',
'test_error_in_setup_and_teardown_verbose.py::test_error_in_teardown UHOH *',
'test_error_in_setup_and_teardown_verbose.py::test_errors_in_setup_and_teardown UHOH *',
])
assert result.ret == 1
result.assert_outcomes(passed=1, skipped=0, failed=0, error=3)
def test_xfail(testdir, xfail_tests):
result = testdir.runpytest(
'--report-failed=f',
'--report-xfailed=✗',
'--report-xpassed=P',
)
result.stdout.fnmatch_lines([
'test_xfail.py P✗f✗✗✗ *',
])
assert result.ret == 1
# TODO: PR for xfail/xpass support in assert_outcomes
result.assert_outcomes(passed=0, skipped=0, failed=1, error=0)
def test_xfail_verbose(testdir, xfail_tests):
result = testdir.runpytest(
'--report-failed-verbose=WTF',
'--report-xfailed-verbose=SHRUGS',
'--report-xpassed-verbose=¯\_(ツ)_/¯',
'-v',
)
result.stdout.fnmatch_lines([
'test_xfail_verbose.py::test_expected_fail_but_passes ¯\_(ツ)_/¯ *',
'test_xfail_verbose.py::test_expected_fail_and_fails SHRUGS *',
'test_xfail_verbose.py::test_expected_fail_but_passes_strict WTF *',
'test_xfail_verbose.py::test_expected_fail_and_fails_strict SHRUGS *',
'test_xfail_verbose.py::test_error_in_setup_and_expected_fail_mark SHRUGS *',
'test_xfail_verbose.py::test_error_in_teardown_and_expected_fail_mark SHRUGS *',
])
assert result.ret == 1
result.assert_outcomes(passed=0, skipped=0, failed=1, error=0)
def test_report_symbols_parsed_from_ini_file(testdir, symbols_in_inifile, one_of_each_result_type):
result = testdir.runpytest()
result.stdout.fnmatch_lines([
'test_report_symbols_parsed_from_ini_file.py ✔🦄✗!?🔥 [100%]',
])
result.assert_outcomes(passed=1, skipped=1, failed=1, error=1)
def test_report_symbols_parsed_from_ini_file_verbose(testdir, symbols_in_inifile, one_of_each_result_type):
result = testdir.runpytest('--verbose')
result.stdout.fnmatch_lines([
'test_report_symbols_parsed_from_ini_file_verbose.py::test_passed OH YEAH [ 16%]',
'test_report_symbols_parsed_from_ini_file_verbose.py::test_xpassed WHAT IN TARNATION? [ 33%]',
'test_report_symbols_parsed_from_ini_file_verbose.py::test_failed OH CRAP [ 50%]',
'test_report_symbols_parsed_from_ini_file_verbose.py::test_xfailed YEAH WHATEVER [ 66%]',
"test_report_symbols_parsed_from_ini_file_verbose.py::test_skipped DON'T CARE [ 83%]",
'test_report_symbols_parsed_from_ini_file_verbose.py::test_error YOU MEDDLING KIDS! [100%]',
])
result.assert_outcomes(passed=1, skipped=1, failed=1, error=1)
outcomes = result.parseoutcomes()
outcomes.pop("seconds") # why is this in here?
assert outcomes == {"passed": 1, "skipped": 1, "failed": 1, "xpassed": 1, "xfailed": 1, "error": 1}
def test_default_symbols():
assert custom_report.symbols == {
(False, 'failed', 'call'): ('failed', 'F', 'FAILED'),
(False, 'failed', 'setup'): ('error', 'E', 'ERROR'),
(False, 'failed', 'teardown'): ('error', 'E', 'ERROR'),
(False, 'passed', 'call'): ('passed', '.', 'PASSED'),
(False, 'skipped', 'call'): ('skipped', 's', 'SKIPPED'),
(False, 'skipped', 'setup'): ('skipped', 's', 'SKIPPED'),
(True, 'passed', 'call'): ('xpassed', 'X', ('XPASS', {'yellow': True})),
(True, 'skipped', 'call'): ('xfailed', 'x', 'xfail'),
(True, 'skipped', 'setup'): ('xfailed', 'x', 'xfail'),
(True, 'skipped', 'teardown'): ('xfailed', 'x', 'xfail'),
}
expected_cli_options = '''\
custom-report:
--report-passed=REPORT_PASSED
symbol to use in the report when a test has passed
(default .)
--report-passed-verbose=REPORT_PASSED_VERBOSE
as above, when '--verbose' is enabled (default PASSED)
--report-xpassed=REPORT_XPASSED
symbol to use when a test is marked as an expected
failure, but it doesn't fail (default X)
--report-xpassed-verbose=REPORT_XPASSED_VERBOSE
as above, when '--verbose' is enabled (default XPASS)
--report-failed=REPORT_FAILED
symbol to use when a test fails, either by assertion
or an unhandled exception (default F)
--report-failed-verbose=REPORT_FAILED_VERBOSE
as above, when '--verbose' is enabled (default FAILED)
--report-xfailed=REPORT_XFAILED
symbol to use when a test fails, but it is marked as
an expected failure (default x)
--report-xfailed-verbose=REPORT_XFAILED_VERBOSE
as above, when '--verbose' is enabled (default xfail)
--report-skipped=REPORT_SKIPPED
symbol to use when a test is skipped, e.g. by
@pytest.skip decorator (default s)
--report-skipped-verbose=REPORT_SKIPPED_VERBOSE
as above, when '--verbose' is enabled (default
SKIPPED)
--report-error=REPORT_ERROR
symbol to use when a test encounters an exception
during setup or teardown (default E)
--report-error-verbose=REPORT_ERROR_VERBOSE
as above, when '--verbose' is enabled (default ERROR)
'''
expected_ini_options = '''\
report_passed (string) symbol to use in the report when a test has passed (default .)
report_passed_verbose (string) as above, when '--verbose' is enabled (default PASSED)
report_xpassed (string) symbol to use when a test is marked as an expected failure, but it doesn't fail (default X)
report_xpassed_verbose (string) as above, when '--verbose' is enabled (default XPASS)
report_failed (string) symbol to use when a test fails, either by assertion or an unhandled exception (default F)
report_failed_verbose (string) as above, when '--verbose' is enabled (default FAILED)
report_xfailed (string) symbol to use when a test fails, but it is marked as an expected failure (default x)
report_xfailed_verbose (string) as above, when '--verbose' is enabled (default xfail)
report_skipped (string) symbol to use when a test is skipped, e.g. by @pytest.skip decorator (default s)
report_skipped_verbose (string) as above, when '--verbose' is enabled (default SKIPPED)
report_error (string) symbol to use when a test encounters an exception during setup or teardown (default E)
report_error_verbose (string) as above, when '--verbose' is enabled (default ERROR)
'''
def test_help_text_contains_cli_options(testdir):
result = testdir.runpytest('--help')
assert expected_cli_options in result.stdout.str()
@pytest.mark.xfail
def test_help_text_contains_ini_options(testdir):
result = testdir.runpytest('--help')
assert expected_ini_options in result.stdout.str()