-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathtest_coverage_check.py
220 lines (180 loc) · 7.93 KB
/
test_coverage_check.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
"""
Unit tests for the CoverageCheck class in the coverage_check module.
"""
import unittest
from subprocess import CompletedProcess
from unittest.mock import patch, MagicMock
from grader.checks.abstract_check import CheckError
from grader.checks.coverage_check import CoverageCheck
class TestCoverageCheck(unittest.TestCase):
"""
Test cases for the CoverageCheck class.
"""
def setUp(self):
"""
Set up the CoverageCheck instance for testing.
"""
self.coverage_check = CoverageCheck("Coverage", 2, "sample_dir")
# This way, we have 3 ranges: 0-33, 34-66, 67-100
return super().setUp()
@patch("subprocess.run")
def test_01_coverage_run_fail(self, mocked_run: MagicMock):
"""
Test that a failed coverage run logs an error and raises an exception.
"""
# Arrange
mocked_run.return_value = CompletedProcess(args=["coverage", "run"], returncode=1)
# Act
with self.assertLogs("grader", level="ERROR") as log:
with self.assertRaises(CheckError):
self.coverage_check.run()
is_message_logged = "ERROR:grader:Coverage run failed" in log.output
# Assert
self.assertTrue(is_message_logged)
@patch("subprocess.run")
def test_02_coverage_report_fail(self, mocked_run: MagicMock):
"""
Test that a failed coverage report logs an error and returns a score of 0.0.
"""
# Arrange
def mocked_run_side_effect(*args, **kwargs):
if "run" in args[0]:
return CompletedProcess(args=["coverage", "run"], returncode=0)
if "report" in args[0]:
return CompletedProcess(args=["coverage", "report"], returncode=1)
mocked_run.side_effect = mocked_run_side_effect
# Act
with self.assertLogs("grader", level="ERROR") as log:
with self.assertRaises(CheckError):
self.coverage_check.run()
is_message_logged = "ERROR:grader:Coverage report failed" in log.output
# Assert
self.assertTrue(is_message_logged)
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_run")
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_report")
def test_03_translate_score_zero(self, mocked_report: MagicMock, mocked_run: MagicMock):
"""
Test that a coverage report of 0 translates to a score of 0.
"""
# Arrange
mocked_run.return_value = True
mocked_report.return_value = 0
expected_score = 0
# Act
actual_score = self.coverage_check.run()
# Assert
self.assertEqual(expected_score, actual_score)
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_run")
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_report")
def test_04_translate_score_inside_first_range(self, mocked_report: MagicMock, mocked_run: MagicMock):
"""
Test that a coverage report inside the first range translates to a score of 0.
"""
# Arrange
mocked_run.return_value = True
mocked_report.return_value = 22
expected_score = 0
# Act
actual_score = self.coverage_check.run()
# Assert
self.assertEqual(expected_score, actual_score)
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_run")
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_report")
def test_05_translate_score_right_bound_first_range(self, mocked_report: MagicMock, mocked_run: MagicMock):
"""
Test that a coverage report at the right bound of the first range translates to a score of 1.
"""
# Arrange
mocked_run.return_value = True
mocked_report.return_value = 100 / 3
expected_score = 1
# Act
actual_score = self.coverage_check.run()
# Assert
self.assertEqual(expected_score, actual_score)
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_run")
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_report")
def test_06_translate_score_left_bound_second_range(self, mocked_report: MagicMock, mocked_run: MagicMock):
"""
Test that a coverage report at the left bound of the second range translates to a score of 1.
"""
# Arrange
mocked_run.return_value = True
mocked_report.return_value = 100 / 3 + 1
expected_score = 1
# Act
actual_score = self.coverage_check.run()
# Assert
self.assertEqual(expected_score, actual_score)
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_run")
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_report")
def test_07_translate_score_inside_bound_second_range(self, mocked_report: MagicMock, mocked_run: MagicMock):
"""
Test that a coverage report inside the second range translates to a score of 1.
"""
# Arrange
mocked_run.return_value = True
mocked_report.return_value = 50
expected_score = 1
# Act
actual_score = self.coverage_check.run()
# Assert
self.assertEqual(expected_score, actual_score)
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_run")
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_report")
def test_08_translate_score_right_bound_second_range(self, mocked_report: MagicMock, mocked_run: MagicMock):
"""
Test that a coverage report at the right bound of the second range translates to a score of 2.
"""
# Arrange
mocked_run.return_value = True
mocked_report.return_value = 100 / 3 * 2
expected_score = 2
# Act
actual_score = self.coverage_check.run()
# Assert
self.assertEqual(expected_score, actual_score)
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_run")
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_report")
def test_09_translate_score_inside_bound_third_range(self, mocked_report: MagicMock, mocked_run: MagicMock):
"""
Test that a coverage report inside the third range translates to a score of 2.
"""
# Arrange
mocked_run.return_value = True
mocked_report.return_value = 75
expected_score = 2
# Act
actual_score = self.coverage_check.run()
# Assert
self.assertEqual(expected_score, actual_score)
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_run")
@patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_report")
def test_10_translate_score_max(self, mocked_report: MagicMock, mocked_run: MagicMock):
"""
Test that a coverage report of 100 translates to a score of 2.
"""
# Arrange
mocked_run.return_value = True
mocked_report.return_value = 100
expected_score = 2
# Act
actual_score = self.coverage_check.run()
# Assert
self.assertEqual(expected_score, actual_score)
@patch("subprocess.run")
def test_11_coverage_report_read_properly(self, mocked_run: MagicMock):
"""
Test that the coverage report is read properly and returns the correct score.
"""
# Arrange
def mocked_run_side_effect(*args, **kwargs):
if "run" in args[0]:
return CompletedProcess(args=["coverage", "run"], returncode=0)
if "report" in args[0]:
return CompletedProcess(args=["coverage", "report"], returncode=0, stdout="100")
mocked_run.side_effect = mocked_run_side_effect
# Act
result = self.coverage_check.run()
# Assert
self.assertEqual(2, result)